diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..6ec44f0d2 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name, const std::shared_ptr& schema, const std::vector& primary_keys = {}) { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr catalog, + std::shared_ptr catalog, AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); PAIMON_RETURN_NOT_OK( AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..36295493f 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,16 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Each of those entry points builds its context the way it does for a data table: from the table's +location, through the builder's usual constructor: + +* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string&)`` + +The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or +``NewCommit()`` - which serves a table whose schema does not live under its own location as well, +such as one a REST catalog serves: the table carries its schema and the file system it was loaded +through, so nothing under the location has to answer for them. Options given at the call still win +over the ones the table carries, as they do everywhere else. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 2d789f839..162799511 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -86,7 +86,7 @@ registered on the REST server. The catalog is configured through the {"token.provider", "bear"}, {"token", ""}, }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(/*root_path=*/"my_instance", options)); For DLF, configure one credential source. Static AK/SK credentials, an optional @@ -127,22 +127,19 @@ without an atomic snapshot-file rename: .. code-block:: cpp - // Writers and committers share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); // CatalogUuid() stays null when the server supplies no id. if (std::optional table_uuid = table->CatalogUuid()) { builder.WithTableId(table_uuid.value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); Configure native writers with the same catalog and identifier. They load the @@ -152,7 +149,7 @@ offsets when a real-time writer is created: .. code-block:: cpp paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, @@ -170,9 +167,9 @@ current snapshot. Both builders use the file system of the table being written or committed for manifests and data, so a catalog that issues temporary credentials per table serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides -it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +it; both also allow ``WithFileSystemSchemeToIdentifierMap`` to route by URI +scheme instead, which the catalog is then not asked for. A format table is +written through the table's own ``NewWrite`` instead; see :doc:`format_table`. A file system passed to ``WithFileSystem`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom @@ -188,7 +185,7 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithFileSystem(my_fs); The snapshot being committed carries a uuid generated on the client, and the @@ -319,18 +316,15 @@ for that table's file system and passing it in through ``WithFileSystem``: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, paimon::TableScan::Create(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, @@ -342,10 +336,9 @@ snapshots, manifests and data under the table path, so it signs every access wit the table's credentials and reloads them as they expire. An explicit ``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. For a format -table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected because the table already carries the file -system it was loaded through. +also how you read with the catalog's own object-store credentials. A format table +is read through the table's own ``NewRead`` or ``NewScan`` instead; see +:doc:`format_table`. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..7579b9dff 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,37 +40,43 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. - -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and +A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table: +its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, +through the ``Table`` interface the two kinds of table share. + +Reading and writing go through the table itself: a caller holding one calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds from +the table's location. This is what Java serves through ``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); - paimon::ScanContextBuilder scan_builder(table); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded, + catalog->GetTable(paimon::Identifier("db", "tbl"))); + // The table's `type` option is `format-table`, so this is a `FormatTable`. + std::shared_ptr table = + std::dynamic_pointer_cast(loaded); + paimon::ScanContextBuilder scan_builder(table->Location()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options +Because the operator comes from the table itself, this serves a table whose schema does not live +under its own location as well, such as one a REST catalog serves: the table carries its schema and +the file system it was loaded through, so nothing under the location has to answer for them. Options given at the call still win over the ones the schema stored, as they do everywhere else. +The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` - serve a data table only: a load from a path is the data table's +way in, and a format table the schema under the path names is refused with a message pointing at +the catalog and the table's own operators above. + +``FormatTable`` is a derived class of ``Table``, so a caller holding one through a +``std::shared_ptr`` calls the same operators on it without the cast: they come back +for the table it is, whatever kind it turns out to be. This is the C++ side of what Java serves +through one ``Table.newReadBuilder()`` for both kinds of table. + A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for and in what order is its own business: they are matched to the table's by name, as Java matches @@ -89,12 +95,10 @@ outside the projection is never read, which is how Java scopes the same checks. named here only to describe what happens. A caller never needs them: a plan comes back as ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``. -A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the -protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say -its metadata lives under the table location, and the REST catalog to take the location and the -schema from one response instead of two that could disagree. A catalog that does not override it -still serves format tables, by reading the location and the schema through the virtuals every -catalog has. +Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the +schema under the table location, so it says the location carries the table's own metadata and a +scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the +location and the schema from one response, so the two cannot disagree. ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by listing directories; every other table type returns ``NotImplemented`` for now. diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..e911bedb1 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, {paimon::Options::FILE_SYSTEM, "local"}}; // create table - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(root_path, options)); PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false)); arrow::FieldVector fields = { diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..3898a1967 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -40,7 +40,6 @@ namespace paimon { using Instant = std::variant; class Database; -class FormatTable; class Table; class View; class Schema; @@ -69,8 +68,11 @@ class PAIMON_EXPORT Catalog { /// schemes, including the file systems it builds from per-table data /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. - /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. - static Result> Create( + /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. + /// @note Shared rather than unique ownership: a global system table of the `sys` database + /// builds its rows lazily through the catalog it was handed, so the table keeps the + /// catalog alive. A catalog built by this factory always satisfies that. + static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, const std::map& fs_scheme_to_identifier_map = {}); @@ -148,6 +150,17 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// + /// The table's `type` option decides what is handed back: a `format-table` comes back as the + /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and + /// every other implemented type as the managed table it is, while a `T$system` identifier names + /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and + /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. + /// + /// A global system table of the "sys" database reads its rows lazily through the catalog it + /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands + /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as + /// through a `std::unique_ptr`, cannot be kept and is refused by name. + /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; @@ -246,33 +259,6 @@ class PAIMON_EXPORT Catalog { /// @note Implementations returning true must also implement the internal /// `VersionManagedCatalog`. virtual bool SupportsVersionManagement() const; - - /// Gets a format table: a directory of data files laid out like a standard Hive table. - /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. - /// - /// @param identifier Identifier of the table to get. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - Result> GetFormatTable(const Identifier& identifier) const; - - protected: - /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back. - /// - /// The default reads the location and the schema through the virtuals above - two requests - /// that can disagree - and treats every directory below the location as table content. An - /// override answers both questions directly: whether location and schema come back in one - /// round trip, and whether this catalog keeps the table's metadata under the table path. - /// - /// @param identifier Identifier of the table to load. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - virtual Result> LoadFormatTable( - const Identifier& identifier) const; }; } // namespace paimon diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..9164623c0 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,50 +33,45 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; /// A table provides basic abstraction for table type. +/// +/// The state every kind of table carries - file system, root path, schema and identifier - lives +/// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and its friends load the table a context names and hand the +/// context to it. A table refuses by name whatever a context asks for that it cannot honour, +/// rather than dropping it silently. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// Returns the file system the table was opened with, null when the opener had none. A + /// catalog that hands out per-table file systems opens a table with the one for that table. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. + /// Returns the root path of the table in its file system. It is empty when the opener had no + /// location for the table, which a mock in a test is one case of. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// A name to identify this table. A managed or format table answers the object name of its + /// identifier, a system table its own name. + virtual std::string Name() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } + /// Full name of the table: `database.tableName` for a managed or format table, the same name + /// as `Name()` for a system table. + virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. @@ -95,37 +90,53 @@ class PAIMON_EXPORT Table { return schema_; } - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. + /// Copies this table with `dynamic_options` on top of the options it carries. An option given + /// at the call wins over the one the schema stored, the precedence every context builder + /// promises. `type` is not overridable: it is structural and read from the schema alone. + /// + /// @param dynamic_options Options given at the call. + /// @return A result containing the copied table, or the error copying it reports. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + + /// Creates a scan of this table from a finished scan context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; + virtual Result> NewScan( + std::unique_ptr context) const = 0; - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. + /// Creates a read of this table from a finished read context. /// @param context The finished read context. /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; + virtual Result> NewRead( + std::unique_ptr context) const = 0; - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. + /// Creates a write of this table from a finished write context. /// @param context The finished write context. /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; + virtual Result> NewWrite( + std::unique_ptr context) const = 0; - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. + /// Creates a commit of this table from a finished commit context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = ""); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..688b10ff9 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -32,7 +32,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -48,8 +47,7 @@ class PAIMON_EXPORT CommitContext { bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options); CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,14 +57,26 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options, const std::string& branch, + const std::optional& table_schema = std::nullopt, + const std::map& fs_scheme_to_identifier_map = {}); ~CommitContext(); const std::string& GetRootPath() const { return root_path_; } + /// Returns the table root path. An alias of `GetRootPath()`, so that every context + /// exposes the table location under the same name. + const std::string& GetPath() const { + return GetRootPath(); + } + + /// Returns the branch the builder resolved out of the identifier and the options. + const std::string& GetBranch() const { + return branch_; + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -79,6 +89,11 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + /// Returns the table schema the caller set, or nullopt when the commit loads it. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + /// Returns the configured catalog, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -110,18 +125,17 @@ class PAIMON_EXPORT CommitContext { return specific_file_system_; } - const std::map& GetOptions() const { - return options_; + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; + const std::map& GetOptions() const { + return options_; } private: std::string root_path_; + std::string branch_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; @@ -132,8 +146,9 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::map options_; - std::shared_ptr format_table_; + std::optional table_schema_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. @@ -144,27 +159,18 @@ class PAIMON_EXPORT CommitContextBuilder { /// @param commit_user The user identifier for the commit operation. CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); - ~CommitContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. CommitContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. @@ -237,6 +243,34 @@ class PAIMON_EXPORT CommitContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system + /// identifiers. This allows selecting different pre-registered file system implementations + /// based on the URI scheme at runtime. + /// + /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding + /// file system identifier. + /// @return Reference to this builder for method chaining. + /// @note + /// - This method is intended for environments where multiple file systems are pre-registered. + /// - The specified identifiers must correspond to file systems that have been registered at + /// compile time or initialization. + /// - Cannot be used together with `WithFileSystem()`. + /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). + /// Example: + /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); + /// + CommitContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main + /// branch, and loaded from the table path otherwise. + CommitContextBuilder& SetTableSchema(const std::string& table_schema); + /// Build and return a `CommitContext` instance with input validation. /// @return Result containing the constructed `CommitContext` or an error status. Result> Finish(); diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..b0fcb17e1 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,7 +50,9 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. + /// Create an instance of `FileStoreCommit`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewCommit` instead. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 8859e4f3f..351b98b06 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,7 +42,9 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. + /// Create an instance of `FileStoreWrite`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewWrite` instead. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e62386c65..2ee575c4f 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -38,7 +38,6 @@ namespace paimon { class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; class FileSystem; @@ -52,23 +51,25 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, bool enable_predicate_filter, - bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, - uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier); + ReadContext(const std::string& path, const std::string& branch, + const std::vector& read_field_names, + const std::vector& read_field_ids, + const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, + bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, + uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, + bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, + const std::optional& table_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, + const std::map& options, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + WarmupLevel warmup_level, const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -164,12 +165,6 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } @@ -184,6 +179,13 @@ class PAIMON_EXPORT ReadContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a read carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -225,7 +227,6 @@ class PAIMON_EXPORT ReadContext { bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; std::shared_ptr catalog_; std::optional identifier_; @@ -240,15 +241,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param path The root path of the table. explicit ReadContextBuilder(const std::string& path); - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); - ~ReadContextBuilder(); ReadContextBuilder(ReadContextBuilder&&) noexcept; @@ -325,14 +317,15 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 6fbb006f0..30f87baeb 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -37,7 +37,6 @@ class ScanContextBuilder; class ScanFilter; class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -57,10 +56,9 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& specific_file_system, const std::optional& table_schema, const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, - const std::optional& identifier); + const std::shared_ptr& cache, const std::shared_ptr& catalog, + const std::optional& identifier, const std::string& branch, + const std::map& fs_scheme_to_identifier_map = {}); ~ScanContext(); @@ -68,6 +66,11 @@ class PAIMON_EXPORT ScanContext { return path_; } + /// Returns the branch the builder resolved out of the identifier and the options. + const std::string& GetBranch() const { + return branch_; + } + bool IsStreamingMode() const { return is_streaming_mode_; } @@ -103,6 +106,10 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; + } + const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -111,12 +118,6 @@ class PAIMON_EXPORT ScanContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -127,8 +128,16 @@ class PAIMON_EXPORT ScanContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a scan carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string path_; + std::string branch_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; @@ -137,10 +146,10 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -178,14 +187,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @param path The root path of the table. explicit ScanContextBuilder(const std::string& path); - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); - ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. ScanContextBuilder& SetLimit(int32_t limit); @@ -208,11 +209,12 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. + /// Add a single configuration option, served the way `SetOptions()` describes. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. @@ -238,6 +240,25 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system + /// identifiers. This allows selecting different pre-registered file system implementations + /// based on the URI scheme at runtime. + /// + /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding + /// file system identifier. + /// @return Reference to this builder for method chaining. + /// @note + /// - This method is intended for environments where multiple file systems are pre-registered. + /// - The specified identifiers must correspond to file systems that have been registered at + /// compile time or initialization. + /// - Cannot be used together with `WithFileSystem()`. + /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). + /// Example: + /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); + /// + ScanContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + /// Plans a native table through its own file system - including the per-table temporary /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..c5bfe60d3 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -47,13 +48,16 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// Reading and writing go through the entry points every other table uses: `TableScan`, -/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. +/// A format table is a `Table`, so it is reached both ways: through the `NewXxx` operators below, +/// and through the entry points every other table uses - `TableScan::Create()` and friends - which +/// recognise one from the schema under the table path, or take one already loaded through the +/// `FormatTable` constructor every context builder has, how a table whose schema lives in a +/// metastore is reached. +/// +/// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. enum class Format { @@ -72,7 +76,7 @@ class PAIMON_EXPORT FormatTable { /// /// This needs a schema file under the table directory, which a table created through /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetFormatTable()` instead. + /// none, and is loaded through `Catalog::GetTable()` instead. /// /// @param file_system File system holding the table directory. /// @param table_path Root path of the table, which is also its data location. @@ -102,24 +106,23 @@ class PAIMON_EXPORT FormatTable { bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. - /// - /// @param table The table to copy. - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or an error status. - static Result> Copy( - const std::shared_ptr& table, - const std::map& dynamic_options); + ~FormatTable() override; - ~FormatTable(); + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + + /// The same table comes back when there is nothing to add. + Result> Copy( + const std::map& dynamic_options) const override; /// Directory the data files live in. const std::string& Location() const { - return location_; + return GetRootPath(); } /// Format of every data file in the directory. @@ -155,27 +158,15 @@ class PAIMON_EXPORT FormatTable { return options_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, database.tableName. - std::string FullName() const; - - /// Schema of the table, including its partition fields. + /// Schema of the table, including its partition fields. Hides `Table::LatestSchema()` to hand + /// back the same schema as the `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { - return schema_; + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. Result> GetArrowSchema() const; - /// File system holding the table directory. - std::shared_ptr GetFileSystem() const { - return file_system_; - } - /// Whether this table's own metadata lives under its location, as told by whoever loaded it. /// /// Only then are the `schema` and `branch` directories below the location table metadata @@ -185,6 +176,19 @@ class PAIMON_EXPORT FormatTable { return location_carries_paimon_metadata_; } + /// The four operators below create this table's scan, read, write and commit, applying the + /// context's options on top of the ones this table carries and refusing by name what a + /// format table cannot honour. + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + private: FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, @@ -192,10 +196,17 @@ class PAIMON_EXPORT FormatTable { const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; + /// This table as a `shared_ptr`. `shared_from_this()` on a const method hands back a + /// `shared_ptr`, while the copy below needs a mutable one; the cast is + /// safe because the table is never written to. + std::shared_ptr SharedSelf() const { + return std::const_pointer_cast(shared_from_this()); + } + + /// `Copy()` as a `FormatTable` rather than a `Table`, which is what its operators need. + Result> CopySelf( + const std::map& dynamic_options) const; + std::map options_; Format format_; std::string file_compression_; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..fa1aef730 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,7 +42,9 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. + /// Create an instance of `TableRead`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewRead` instead. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..edecb59c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,7 +36,9 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. + /// Create an instance of `TableScan`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewScan` instead. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..d34bfe257 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -33,7 +33,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -53,7 +52,7 @@ class PAIMON_EXPORT WriteContext { const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, - const std::shared_ptr& format_table, + const std::optional& table_schema = std::nullopt, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -63,6 +62,12 @@ class PAIMON_EXPORT WriteContext { return root_path_; } + /// Returns the table root path. An alias of `GetRootPath()`, so that every context + /// exposes the table location under the same name. + const std::string& GetPath() const { + return GetRootPath(); + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -124,10 +129,9 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; + /// Returns the table schema the caller set, or nullopt when the write loads it. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; } /// Returns the catalog supplying table metadata, or null if unset. @@ -140,6 +144,13 @@ class PAIMON_EXPORT WriteContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a write carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string root_path_; std::string commit_user_; @@ -157,7 +168,7 @@ class PAIMON_EXPORT WriteContext { std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; std::map options_; - std::shared_ptr format_table_; + std::optional table_schema_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -170,28 +181,18 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param commit_user The user identifier for commit operations. WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); - ~WriteContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. WriteContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. @@ -300,6 +301,15 @@ class PAIMON_EXPORT WriteContextBuilder { WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main + /// branch, and loaded from the table path otherwise. + WriteContextBuilder& SetTableSchema(const std::string& table_schema); + /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. /// If > 0, sets arrow CPU thread pool capacity for spill operations. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b4ad6e80f..d2da868ea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -267,6 +267,7 @@ set(PAIMON_CORE_SRCS core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp + core/catalog/snapshot_commit.cpp core/core_options.cpp core/deletionvectors/deletion_vector.cpp core/deletionvectors/deletion_file_writer.cpp @@ -443,8 +444,11 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp - core/table/table.cpp + core/table/append_only_file_store_table.cpp core/table/bucket_mode.cpp + core/table/file_store.cpp + core/table/file_store_table.cpp + core/table/file_store_table_factory.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -452,11 +456,11 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_commit.cpp core/table/format/format_table_file_store_commit.cpp core/table/format/format_table_file_store_write.cpp - core/table/format/format_table_loader.cpp core/table/format/format_table_read.cpp core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp core/table/format/lazy_concat_batch_reader.cpp + core/table/primary_key_file_store_table.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp @@ -490,6 +494,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp + core/table/table.cpp core/tag/tag.cpp core/utils/branch_manager.cpp core/utils/blob_view_lookup.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..717606612 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -26,11 +26,9 @@ #include "fmt/format.h" #include "paimon/catalog_options.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" #ifdef PAIMON_ENABLE_REST #include "paimon/rest/rest_catalog.h" #endif @@ -42,7 +40,7 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create( +Result> Catalog::Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system, const std::map& fs_scheme_to_identifier_map) { @@ -55,8 +53,11 @@ Result> Catalog::Create( } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), - fs_scheme_to_identifier_map); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr rest_catalog, + RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map)); + return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -67,26 +68,12 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::shared_ptr( + std::make_unique(core_options.GetFileSystem(), root_path, options)); } bool Catalog::SupportsVersionManagement() const { return false; } -Result> Catalog::GetFormatTable(const Identifier& identifier) const { - // Refused here rather than in each override, so every catalog rejects a system table the same - // way and in the same order: the database first, so that an identifier of "sys" is rejected - // without being parsed as a table name. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable")); - return LoadFormatTable(identifier); -} - -Result> Catalog::LoadFormatTable(const Identifier& identifier) const { - // The location and the schema are read separately, and every directory below the location is - // table content: a catalog that knows better overrides this and says so. - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/false); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..4cf128a94 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -22,16 +22,17 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -44,6 +45,21 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) } // namespace +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { return db_name == Catalog::SYSTEM_DATABASE_NAME; } @@ -75,6 +91,16 @@ Result CatalogUtils::BranchIdentifier(const Identifier& identifier, return Identifier(identifier.GetDatabaseName(), data_table_name, branch); } +Result> CatalogUtils::BranchIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); if (branch) { @@ -117,12 +143,6 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, } PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(data_schema->Options())); - if (table_type == TableType::FORMAT_TABLE) { - return Status::Invalid( - fmt::format("Cannot open format table '{}' as a Table in '{}', please use " - "'Catalog::GetFormatTable' or 'FormatTable::Create'.", - identifier.ToString(), action)); - } // A materialized table is a managed table that also carries the SQL it materializes. if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) { const std::map& options = data_schema->Options(); @@ -136,17 +156,4 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, return Status::OK(); } -Result> CatalogUtils::LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) { - PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, catalog.LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table", - identifier.GetFullName())); - } - return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema, - location_carries_paimon_metadata, /*dynamic_options=*/{}); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..58775d4df 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "paimon/catalog/identifier.h" @@ -31,13 +32,19 @@ class Catalog; class FormatTable; class Schema; -/// Checks shared by the catalog implementations. Every check takes an `action` naming -/// the rejected operation in the error message, e.g. "dropTable". +/// The identifier tools and the checks shared by the catalog implementations. Every check takes an +/// `action` naming the rejected operation in the error message, e.g. "dropTable". class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; + /// The identifier a table path names when no catalog name goes with it, the C++ reading of the + /// Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the + /// directory above the table's names the database when it carries the `.db` suffix a catalog + /// lays its databases out with, and a path that names no database leaves it unknown. + static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); + /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); @@ -58,6 +65,12 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); + /// The identifier a catalog knows the table by, on `branch`, when there is one to fold a + /// branch into: a caller without an identifier has none to ask the catalog about, and gets + /// none back rather than an error. + static Result> BranchIdentifier( + const std::optional& identifier, const std::string& branch); + /// Fails when `db_name` cannot be used as a single path component, which is required to /// keep the database path under the warehouse. static Status CheckValidDatabaseName(const std::string& db_name); @@ -66,32 +79,19 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a `Table` can describe. + /// Fails when `schema` is not a table a managed `Table` can describe. /// - /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the - /// remaining table types are stored differently and are not implemented here, so handing one - /// back as a `Table` would promise snapshots it never had. A schema that is not a data - /// table's, such as a system table's, passes. + /// Only a managed table is one: the remaining table types are stored differently and are + /// not implemented here, so handing one back as a `Table` would promise snapshots it never + /// had. A schema that is not a data table's, such as a system table's, passes. A format + /// table is served by the caller - `GetTable()` hands one back as the `FormatTable` it is - + /// so its schema only reaches this check by mistake and falls to the refusal below. /// /// @param action The call being refused, qualified as a caller would write it, e.g. /// `Catalog::GetTable`. It names the entry point in the message. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); - - /// Loads `identifier` as a format table by reading its location and its schema separately. - /// - /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A - /// catalog that can get both from one response should not use it, since two requests can - /// disagree. - /// - /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata - /// under the table path, which is what tells a `schema` or `branch` directory below the - /// location from a partition directory of the same name. See - /// `FormatTable::LocationCarriesPaimonMetadata()`. - static Result> LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, - bool location_carries_paimon_metadata); }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..656a7e39c 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -34,7 +34,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -44,6 +46,7 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" +#include "paimon/table/format/format_table.h" namespace arrow { class Schema; @@ -285,46 +288,22 @@ Result> FileSystemCatalog::LoadTableSchema( return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); + return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - PAIMON_ASSIGN_OR_RAISE( - Identifier data_identifier, - CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - TableSchemaExists(data_identifier)); - if (!latest_schema) { - return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); - } - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + LoadSystemTable(identifier)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -335,15 +314,28 @@ Result> FileSystemCatalog::LoadTableSchema( } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +345,49 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } // Only a managed table is a `Table`; describing another type as one would hand back // snapshots and manifests it never had, so the failure belongs to the call that opens the // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); +} + +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -579,10 +606,4 @@ Result> FileSystemCatalog::GetLatestSnapshot( return std::optional(snapshot->ToSnapshotInfo()); } -Result> FileSystemCatalog::LoadFormatTable( - const Identifier& identifier) const { - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/true); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..0f3123238 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -37,8 +37,9 @@ class TableSchema; class FileSystem; class Identifier; class Logger; +class SystemTable; -class FileSystemCatalog : public Catalog { +class FileSystemCatalog : public Catalog, public std::enable_shared_from_this { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, const std::map& catalog_options); @@ -71,13 +72,6 @@ class FileSystemCatalog : public Catalog { Result> GetLatestSnapshot(const Identifier& identifier, const std::string& branch) const override; - protected: - /// This catalog keeps a table's schema under the table's own path, so the `schema` and - /// `branch` directories there are metadata rather than table content. Nothing else differs, - /// so the location and the schema are still read separately. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. @@ -92,6 +86,10 @@ class FileSystemCatalog : public Catalog { Result>> TableSchemaExists( const Identifier& identifier) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries. The base table it belongs to must exist and carry a schema. + Result> LoadSystemTable(const Identifier& identifier) const; + Status CreateDatabaseImpl(const std::string& db_name, const std::map& options); Result TableExistsInFileSystem(const std::string& table_path) const; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 51e7c642d..cc8fea8d8 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -34,14 +34,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -223,10 +230,13 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - ASSERT_EQ(system_table->Name(), "tbl1$options"); - // A system table reads the files of the data table it belongs to, so the table opened - // for it carries that table's root path. + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); + ASSERT_EQ("tbl1$options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); + ASSERT_EQ((std::vector{"key", "value"}), + system_table->LatestSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -242,6 +252,129 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } +TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); + + ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_EQ("tbl1$options", system_table->Name()); + + ScanContextBuilder scan_builder(system_table->GetRootPath()); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, system_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + + ReadContextBuilder read_builder(system_table->GetRootPath()); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, system_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 2); + + WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); + write_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), + "tbl1$options system table is read-only: writing is not supported"); + + CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); + commit_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG( + system_table->NewCommit(std::move(commit_context)), + "tbl1$options system table is read-only: committing for it is not supported"); +} + +TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + { + std::map disabled = options; + disabled.erase(CatalogOptionsSystemTable::kEnabledOption); + auto disabled_catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), disabled); + ASSERT_NOK_WITH_MSG(disabled_catalog->GetTable(Identifier("sys", "catalog_options")), + "not exist"); + } + ASSERT_OK_AND_ASSIGN(auto catalog_options, + catalog->GetTable(Identifier("sys", "catalog_options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); + + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); + + ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); + ScanContextBuilder scan_builder("sys/tables"); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, tables->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + + ReadContextBuilder read_builder("sys/tables"); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, tables->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 1); + + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + FileSystemCatalog stack_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_NOK_WITH_MSG(stack_catalog.GetTable(Identifier("sys", "tables")), + "must be owned by a std::shared_ptr"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -629,7 +762,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -650,11 +783,9 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, - catalog.GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); + std::shared_ptr format_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, format_table); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); ASSERT_EQ(format_table->FullName(), "db1.fmt"); @@ -662,15 +793,6 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { // `branch` directories there are its metadata and a scan leaves them alone. A catalog that // keeps schemas elsewhere says no, and then every directory below the location is data. ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata()); - - // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers - // alike. The database is checked before the table name is parsed, so a malformed name under - // "sys" is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); } TEST(FileSystemCatalogTest, TestInvalidCreateTable) { diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp new file mode 100644 index 000000000..c9fac382b --- /dev/null +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/snapshot_commit.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/defs.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +Result> SnapshotCommit::Create( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const CoreOptions& options, const std::optional& table_id, + const std::string& commit_user, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager) { + if (AsVersionManaged(catalog) != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, + AsVersionManaged(catalog)->CreateCommitLock( + catalog_identifier.value(), table_id, commit_user, options)); + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + catalog, catalog_identifier.value(), table_id, std::move(lock))); + } + if (use_rest_catalog_commit) { + if (options.RestCommitAcquireLock()) { + return Status::Invalid(fmt::format( + "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); + } + return std::shared_ptr(std::make_shared(table_id)); + } + if (table_id && catalog == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(file_system, snapshot_manager)); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 41bd1712a..2062b6c2f 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -23,12 +23,18 @@ #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" namespace paimon { +class Catalog; +class CoreOptions; +class FileSystem; +class SnapshotManager; + /// Interface to commit snapshot atomically. class SnapshotCommit { public: @@ -41,6 +47,28 @@ class SnapshotCommit { virtual Result> AcquireLock() { return std::unique_ptr(); } + /// The commit a table publishes its snapshots through, the C++ reading of the Java + /// `CatalogEnvironment#snapshotCommit`: a catalog that version-manages the table commits + /// through it, a caller that only wants the request built gets one with no catalog attached, + /// and everything else publishes by renaming the snapshot file into place. + /// + /// @param catalog The catalog the commit was handed, or null for one that has none. + /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the + /// commit commits to. A version-managing catalog needs one and is refused without it. + /// @param options Table options merged with the writer's dynamic options. + /// @param table_id The catalog's UUID for the table, captured before the changes were + /// prepared; null when there is none. + /// @param commit_user The commit user of the snapshots published under a commit lock. + /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the + /// catalog to send it. + /// @param file_system The file system the snapshot file is renamed through. + /// @param snapshot_manager The manager that names the snapshot file renamed. + static Result> Create( + const std::shared_ptr& catalog, + const std::optional& catalog_identifier, const CoreOptions& options, + const std::optional& table_id, const std::string& commit_user, + bool use_rest_catalog_commit, const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager); /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. /// @param snapshot Snapshot to be committed. diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..85690b25f 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -298,7 +298,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); ASSERT_OK(catalog->CreateDatabase("db", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("db", "table"), &c_schema, @@ -373,7 +373,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsIndexForColumnNamedLikeInter schema_ = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field(kColumnName, arrow::utf8(), /*nullable=*/false)}); options_[Options::PK_BTREE_INDEX_COLUMNS] = kColumnName; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); for (bool spillable : {false, true}) { SCOPED_TRACE(spillable); const std::string table_name = spillable ? "row_id_spill" : "row_id_memory"; diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..ba0713ec0 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -33,7 +33,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -43,25 +42,28 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::map& options) : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} + options, + // Without an identifier the option is the only name a branch can have. + BranchManager::NormalizeBranch( + options.count(Options::BRANCH) ? options.at(Options::BRANCH) : ""), + /*table_schema=*/std::nullopt, /*fs_scheme_to_identifier_map=*/{}) {} -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, - bool append_commit_check_conflict, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) +CommitContext::CommitContext( + const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, const std::shared_ptr& catalog, + const std::optional& identifier, const std::optional& table_id, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& options, const std::string& branch, + const std::optional& table_schema, + const std::map& fs_scheme_to_identifier_map) : root_path_(root_path), + branch_(branch), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), @@ -72,8 +74,9 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), options_(options), - format_table_(format_table) {} + table_schema_(table_schema) {} CommitContext::~CommitContext() = default; @@ -91,15 +94,13 @@ class CommitContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); + fs_scheme_to_identifier_map_.clear(); + table_schema_ = std::nullopt; options_.clear(); } private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; @@ -110,6 +111,8 @@ class CommitContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; + std::optional table_schema_; std::map options_; }; @@ -120,15 +123,6 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, @@ -189,13 +183,25 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem( return *this; } +CommitContextBuilder& CommitContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + +CommitContextBuilder& CommitContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot commit through a null catalog"); } + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } if (impl_->identifier_) { // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` // reads back as `dev`, so a commit built for such a name would be aimed at another branch. @@ -230,12 +236,13 @@ Result> CommitContextBuilder::Finish() { "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the commit uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { @@ -245,7 +252,8 @@ Result> CommitContextBuilder::Finish() { impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->specific_file_system_, impl_->options_, branch, impl_->table_schema_, + impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 51fcab399..bcbef8bcc 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -40,8 +40,10 @@ TEST(CommitContextTest, TestLegacyConstructor) { auto fs = std::make_shared(); const std::map options = {{"key", "value"}}; CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); + options); ASSERT_EQ(context.GetRootPath(), "table_root_path"); + // Without an identifier the option is the only name a branch can have. + ASSERT_EQ(context.GetBranch(), "main"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); ASSERT_TRUE(context.UseRESTCatalogCommit()); @@ -53,6 +55,7 @@ TEST(CommitContextTest, TestLegacyConstructor) { ASSERT_EQ(context.GetCatalog(), nullptr); ASSERT_FALSE(context.GetIdentifier()); ASSERT_FALSE(context.GetTableId()); + ASSERT_TRUE(context.GetFileSystemSchemeToIdentifierMap().empty()); } TEST(CommitContextTest, TestDefaultValue) { @@ -62,6 +65,7 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_TRUE(ctx->IgnoreEmptyCommit()); ASSERT_FALSE(ctx->UseRESTCatalogCommit()); @@ -73,6 +77,28 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetExecutor()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetOptions().empty()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); + ASSERT_FALSE(ctx->GetTableId()); +} + +TEST(CommitContextTest, TestFileSystemAndSchemeMapConflict) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(CommitContextTest, TestSchemeMapWithoutFileSystem) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(CommitContextTest, TestSetContent) { @@ -89,6 +115,7 @@ TEST(CommitContextTest, TestSetContent) { .WithMemoryPool(memory_pool) .WithExecutor(executor) .WithFileSystem(fs) + .SetTableSchema("table-schema-json") .AddOption("key", "value") .Finish()); @@ -102,6 +129,8 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(ctx->GetOptions(), expected_options); @@ -142,6 +171,8 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { auto branch_ctx, branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); + // Finish() stores the resolved branch, so the factory reads it instead of resolving again. + ASSERT_EQ(branch_ctx->GetBranch(), "dev"); CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) .AddOption(Options::BRANCH, "dev") @@ -151,6 +182,7 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { auto constructed_ctx, constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); + ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); CommitContextBuilder main_builder("table_root_path", "commit_user_1"); ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) .AddOption(Options::BRANCH, "main") @@ -193,7 +225,9 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { TEST(CommitContextTest, TestBranch) { CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK(file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_OK_AND_ASSIGN(auto fs_branch_ctx, + file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_EQ(fs_branch_ctx->GetBranch(), "dev"); CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, @@ -207,6 +241,7 @@ TEST(CommitContextTest, TestBranch) { auto rest_ctx, rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); + ASSERT_EQ(rest_ctx->GetBranch(), "dev"); ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4e0747437..9b6c3975c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,3 +18,19 @@ */ #include "paimon/file_store_commit.h" + +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + +namespace paimon { + +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewCommit(std::move(ctx)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..732f83bd0 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -41,6 +41,7 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -100,7 +101,7 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); + BranchManager::DEFAULT_MAIN_BRANCH); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "a catalog commit requires a table identifier"); } diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..ea73b2246 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,8 +18,19 @@ #include "paimon/file_store_write.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/write_context.h" + namespace paimon { +Result> FileStoreWrite::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewWrite(std::move(ctx)); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 520e850e5..ffee33161 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -170,14 +170,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - std::shared_ptr shared_catalog(std::move(catalog)); - ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); + ASSERT_FALSE(catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder.WithCatalog(catalog, Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..e17cfbd4e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/internal_read_context.h" #include +#include #include #include #include @@ -44,7 +45,10 @@ #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" @@ -273,13 +277,69 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } +Result> InternalReadContext::CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string& root_path) { + core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path); +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + return CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt); +} + +Result> InternalReadContext::CreateFromOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path) { + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); +} + +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options) { + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager->Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + table_schema = latest_schema.value(); + } + assert(table_schema); + auto options = table_schema->Options(); + for (const auto& [key, value] : table_dynamic_options) { + if (key != Options::TYPE) { + options[key] = value; + } + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); + } + return CreateFromOptions(context, table_schema, options, file_system, root_path); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, std::optional path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( @@ -390,8 +450,8 @@ Result> InternalReadContext::Create( return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - auto internal_context = std::unique_ptr( - new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + auto internal_context = std::unique_ptr(new InternalReadContext( + context, table_schema, read_schema, core_options, std::move(path), arrow_pool)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); return internal_context; } @@ -471,12 +531,14 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, + std::optional path, const std::shared_ptr& arrow_pool) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -490,7 +552,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_)); + original->options_, original->path_, original->arrow_pool_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..a710ca15e 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class Schema; namespace paimon { struct CatalogContext; class Executor; +class FileSystem; class MemoryPool; class Predicate; class RealtimeContext; @@ -56,11 +58,43 @@ class InternalReadContext { std::string field_name; }; + /// Builds the context of the read a table serves: the options are the table's own, already + /// parsed, and the cache the context carries is armed on them here. + /// @param root_path The root path the table was opened with. The read reads from it in place of + /// the path the context names. + static Result> CreateForTable( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string& root_path); + + /// Builds the context of a read whose options are composed on the spot rather than the table's + /// own: the reads a write or a lookup serve inside themselves. Such a read serves the file + /// system the context resolved and reads from the path the context names. A fallback branch + /// reads through `CreateForFallbackBranch` instead, whose schema belongs to another branch. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::map& options); + /// The read context of a fallback branch, whose schema is not the table's own: it is read + /// through a manager of that branch, and the read serves the options that schema was loaded + /// with, the branch it was read from among them. A branch the context carries is not honoured + /// here, and neither are options the context adds on its own: the fallback reader serves the + /// table the way the main reader does, and a load merges the context's options into the + /// dynamic options handed in here. The C++ reading of the Java + /// `FileStoreTableFactory#createOtherBranchTable`, which loads the other branch's table the + /// same way for the fallback reader to serve. + /// @param file_system The file system the table was opened with, in place of one resolved from + /// the context. + /// @param root_path The root path the table was opened with, in place of the one the context + /// names. + /// @param table_dynamic_options The dynamic options the load merged into the table - and the + /// ones its copies accumulated. + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; @@ -79,7 +113,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -163,9 +197,25 @@ class InternalReadContext { InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, - const CoreOptions& options, + const CoreOptions& options, std::optional path, const std::shared_ptr& arrow_pool); + /// The tail of a read whose options are a map: they resolve to `CoreOptions` against + /// `file_system`, or against the context's own when it is null, and then go where + /// `CreateFromCoreOptions` takes them. + static Result> CreateFromOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path); + + /// The tail both `Create` entries share: the options arrive parsed and cached, and the + /// validations and the read schema below are the same whichever way they were resolved. + static Result> CreateFromCoreOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + std::optional path); + static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search); @@ -184,6 +234,9 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; + /// The root path a table reads through, or empty for the read context's own, which a caller + /// without a table leaves it as. + std::optional path_; }; } // namespace paimon diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 397ba68ea..8fae06e2c 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -33,6 +33,7 @@ #include "paimon/core/operation/orphan_files_cleaner_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -150,15 +151,7 @@ Result> CleanContextBuilder::Finish() { Result> OrphanFilesCleaner::Create( std::unique_ptr&& ctx) { - if (ctx == nullptr) { - return Status::Invalid("clean context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "clean")); if (ctx->GetOlderThanMs() < 0) { return Status::Invalid("older than time needs to be greater than or equal to 0."); } diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bc65b4661..40ed358de 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -32,7 +32,6 @@ #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -50,8 +49,7 @@ ReadContext::ReadContext( const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level, const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), branch_(branch), @@ -77,7 +75,6 @@ ReadContext::ReadContext( read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), warmup_level_(warmup_level), catalog_(catalog), identifier_(identifier) {} @@ -135,10 +132,6 @@ class ReadContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` /// option is not overruled by the default. std::optional branch_; @@ -175,15 +168,6 @@ ReadContextBuilder::ReadContextBuilder(const std::string& path) impl_->path_ = path; } -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; @@ -359,34 +343,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); @@ -436,8 +392,8 @@ Result> ReadContextBuilder::Finish() { impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, - impl_->catalog_, impl_->identifier_); + impl_->cache_config_, impl_->cache_, impl_->warmup_level_, impl_->catalog_, + impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 4fb5e00b4..fd1b1ebc5 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -116,6 +116,7 @@ TEST(ReadContextTest, TestDefaultValue) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..5d1c7610c 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -22,26 +22,30 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) +ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, + std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& options, + const std::shared_ptr& cache, + const std::shared_ptr& catalog, + const std::optional& identifier, const std::string& branch, + const std::map& fs_scheme_to_identifier_map) : path_(path), + branch_(branch), is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), @@ -50,10 +54,10 @@ ScanContext::ScanContext( memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -74,6 +78,7 @@ class ScanContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_.reset(); specific_file_system_.reset(); + fs_scheme_to_identifier_map_.clear(); table_schema_ = std::nullopt; options_.clear(); cache_.reset(); @@ -83,10 +88,6 @@ class ScanContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -98,6 +99,7 @@ class ScanContextBuilder::Impl { // Resolved in Finish(); a builder never owns an executor of its own. std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; std::map options_; std::shared_ptr cache_; @@ -110,15 +112,6 @@ ScanContextBuilder::ScanContextBuilder(const std::string& path) impl_->path_ = path; } -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; @@ -187,6 +180,12 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, const Identifier& identifier) { impl_->catalog_ = catalog; @@ -205,28 +204,6 @@ ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot scan with empty table path"); @@ -234,12 +211,21 @@ Result> ScanContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot scan through a null catalog"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. + // GetTableFileSystem, so the scan uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, + /*explicit_branch=*/std::nullopt, "scan")); std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( @@ -248,7 +234,7 @@ Result> ScanContextBuilder::Finish() { impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->catalog_, impl_->identifier_, branch, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 4ce2395b3..7a0e9e1bc 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -70,16 +70,75 @@ TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } +TEST(ScanContextTest, TestFileSystemAndSchemeMapConflict) { + ScanContextBuilder builder("table_root_path"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(ScanContextTest, TestSchemeMapWithoutFileSystem) { + ScanContextBuilder builder("table_root_path"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); +} + +TEST(ScanContextTest, TestSchemeMapOverridesCatalogFileSystem) { + // A scheme map routes by scheme instead of a per-table file system, so the catalog is not + // asked for one when a map is set. + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); +} + TEST(ScanContextTest, TestWithCatalogNullRejected) { ScanContextBuilder builder("table_root_path"); ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), "cannot scan through a null catalog"); } +TEST(ScanContextTest, TestBranchResolution) { + // Finish() resolves the branch out of the identifier and the options and stores it, so the + // factory reads a resolved branch instead of resolving it on the spot. + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto option_ctx, builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_EQ("dev", option_ctx->GetBranch()); + + auto catalog = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto identifier_ctx, + builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")).Finish()); + ASSERT_EQ("data", identifier_ctx->GetBranch()); + + // Names which disagree are refused rather than silently chosen between. + ScanContextBuilder mismatch_builder("table_root_path"); + ASSERT_NOK_WITH_MSG(mismatch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")) + .AddOption(Options::BRANCH, "rt") + .Finish(), + "a scan is aimed at one branch, but both 'data' and 'rt' were named"); +} + TEST(ScanContextTest, TestDefaultValue) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_EQ("main", ctx->GetBranch()); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); ASSERT_TRUE(ctx->GetMemoryPool()); @@ -91,6 +150,7 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); } TEST(ScanContextTest, TestSetContent) { diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..fbcbfe368 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -22,6 +22,7 @@ #include "arrow/util/thread_pool.h" #include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" @@ -30,7 +31,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -44,7 +44,7 @@ WriteContext::WriteContext( const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::optional& table_schema, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), @@ -62,7 +62,7 @@ WriteContext::WriteContext( fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), options_(options), - format_table_(format_table), + table_schema_(table_schema), catalog_(catalog), identifier_(identifier) {} @@ -88,15 +88,12 @@ class WriteContextBuilder::Impl { realtime_context_.reset(); catalog_.reset(); identifier_.reset(); + table_schema_ = std::nullopt; options_.clear(); } private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or /// the `branch` option is not overruled by the default. @@ -115,6 +112,7 @@ class WriteContextBuilder::Impl { std::shared_ptr realtime_context_; std::shared_ptr catalog_; std::optional identifier_; + std::optional table_schema_; std::map options_; }; @@ -125,15 +123,6 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, @@ -206,6 +195,11 @@ WriteContextBuilder& WriteContextBuilder::SetWriteBufferSpillThreadNumber(int32_ return *this; } +WriteContextBuilder& WriteContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + WriteContextBuilder& WriteContextBuilder::WithFileSystem( const std::shared_ptr& file_system) { impl_->specific_file_system_ = file_system; @@ -226,27 +220,6 @@ WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -276,6 +249,14 @@ Result> WriteContextBuilder::Finish() { branch, table_name, branch)); } } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( @@ -286,7 +267,7 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, + impl_->realtime_context_, impl_->options_, impl_->table_schema_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 5e777c351..a0137bf59 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -51,8 +51,10 @@ TEST(WriteContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); ASSERT_FALSE(ctx->GetCatalog()); ASSERT_FALSE(ctx->GetIdentifier()); + ASSERT_FALSE(ctx->GetTableId()); } TEST(WriteContextTest, TestWithCatalog) { @@ -69,6 +71,28 @@ TEST(WriteContextTest, TestWithCatalog) { "cannot write through a null catalog"); } +TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { + // WithCatalog resolves the per-table file system when Finish() builds the context, which the + // factory then reads instead of asking the catalog again. + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + + WriteContextBuilder builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + + // A scheme map routes by scheme instead, so the catalog is not asked for a file system. + WriteContextBuilder map_builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); +} + TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { auto catalog = std::make_shared(); WriteContextBuilder builder("table_root_path", "commit_user_1"); @@ -172,6 +196,7 @@ TEST(WriteContextTest, TestSetContent) { .WithWriteSchema(write_schema) .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) .WithFileSystem(file_system) + .SetTableSchema("table-schema-json") .AddOption("key", "value") .Finish()); @@ -187,6 +212,8 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 3a73838d9..9fcb5f370 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -76,6 +76,34 @@ namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. They name +/// the physical layout the data on the table's path was written by: a `bucket-key` the options +/// rename, say, re-hashes the bucket pruning while the files stay hashed by the old key, so the +/// scans a copy serves would filter out the buckets its rows are actually in. The check runs over +/// every option the call gives, before the merge, so it holds on every path that merges options +/// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and +/// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped +/// by `Copy` before this point. +// TODO(yonghao.fyh): refuse the rest of Java's immutable set, which this list does not cover yet: +// `blob-descriptor-field` (only its deprecated `blob.stored-descriptor-fields` fallback is above), +// `data-file.path-directory`, `sequence.snapshot-ordering`, `rowkind.field`, `primary-key`, +// `primary-key.nullable`, `partition`, `dynamic-bucket.initial-buckets`, `video-frame-field`, and +// the four servable keys named above, should the C++ side stop serving them as dynamic options. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { return true; @@ -402,6 +430,31 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { return Status::OK(); } +Status SchemaValidation::CheckImmutableOptions( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..ba169c057 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,6 +51,15 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Whether a copy's dynamic options stay within what the data on the table's path was laid out + /// with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`. An option + /// that says what the schema already says moves nothing, an immutable one that says anything + /// else is refused, and `ignore-delete` carries the Java check's directional rule of its own: + /// it may only ever be turned on, never back off. The immutable options themselves are the + /// Java `CoreOptions.IMMUTABLE_OPTIONS`, as far as the C++ side has constants for them. + static Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..e8a3b7b65 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,31 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +357,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..aff4ee920 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -101,6 +101,15 @@ class TableSchema : public DataSchema, public Jsonizable { const std::map& Options() const override { return options_; } + + /// A copy of this schema with `dynamic_options` on top of the options it carries, which is + /// the precedence every copy of a table promises. Everything else - id, fields, keys, + /// comment - is copied as it is. + /// + /// @param dynamic_options Options given at the call, which win over the schema's own. + /// @return A result containing the merged schema, or an error status. + Result> MergeOptions( + const std::map& dynamic_options) const; const std::vector& Fields() const { return fields_; } @@ -142,6 +151,10 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; + /// Derives `bucket_keys_` and `num_bucket_` from the options the schema carries: every + /// constructor path must call this, as both members serve scans and copies. + Status DeriveBucketInfo(); + static Result> AssignFieldIdsRecursively( const std::shared_ptr& field, bool set_field_id, int32_t* field_id); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..77df95336 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, schema, identifier, uuid) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + std::shared_ptr schema = std::static_pointer_cast(schema_); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(options, path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = + std::make_shared(std::move(index_file_handler)); + } + + auto file_store_write = std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..5dee43d11 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema without primary keys describes, Java Paimon's +/// `AppendOnlyFileStoreTable`. Its write appends the rows it is given, and its scan, read and +/// commit walk the files as they were written. The operator flows are the ones `FileStoreTable` +/// carries; the pieces those flows hand down are built here. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds an append-only table through: the store + /// carries the storage state, `schema` is the schema the table was loaded with, and the base + /// class holds both. + AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp new file mode 100644 index 000000000..157b807f4 --- /dev/null +++ b/src/paimon/core/table/file_store.cpp @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +std::shared_ptr FileStore::Create(const std::string& root_path, + CoreOptions core_options) { + return std::shared_ptr(new FileStore(root_path, std::move(core_options))); +} + +FileStore::FileStore(const std::string& root_path, CoreOptions core_options) + : file_system_(core_options.GetFileSystem()), + root_path_(root_path), + branch_(BranchManager::NormalizeBranch(core_options.GetBranch())), + schema_manager_(std::make_shared(file_system_, root_path_, branch_)), + snapshot_manager_(std::make_shared(file_system_, root_path_, branch_)), + core_options_(std::move(core_options)) {} + +Result> FileStore::NewPathFactory( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path_, arrow_schema, partition_keys, options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +} + +Result> FileStore::NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const { + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path_, branch_); + if (VersionManagedCatalog* versioned = AsVersionManaged(catalog)) { + if (!catalog_identifier) { + return Status::Invalid("a version-managed catalog requires a table identifier"); + } + Identifier identifier = catalog_identifier.value(); + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + return snapshot_manager; +} + +Result> FileStore::NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + memory_pool)); + return manifest_list; +} + +Result> FileStore::NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), memory_pool, options, + partition_schema)); + return manifest_file; +} + +Result> FileStore::NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + return IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), memory_pool, options); +} + +Result> FileStore::NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + NewIndexManifestFile(options, path_factory, memory_pool)); + return std::make_unique( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), options.DeletionVectorsBitmap64(), + memory_pool); +} + +Result> FileStore::NewPartitionComputer( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::shared_ptr& memory_pool) const { + return BinaryRowPartitionComputer::Create(partition_keys, arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h new file mode 100644 index 000000000..58812a76e --- /dev/null +++ b/src/paimon/core/table/file_store.h @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class BinaryRowPartitionComputer; +class Catalog; +class IndexFileHandler; +class IndexManifestFile; +class ManifestFile; +class ManifestList; + +/// Holds the storage a managed table reads and writes through: the file system, root path, +/// branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. +/// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. +/// +/// The factories below take the options of the call, so an operator can serve a cache of its own +/// over the options the table was loaded with. Each call builds a fresh instance and never a +/// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. +class FileStore { + public: + /// Builds the store over options resolved before the call - `FileStoreTable::Create` resolves + /// them once - so the store consumes the resolution and parses nothing itself. + /// @param root_path Root path of the table in the file system. + /// @param core_options The final options the table serves. The store reads and writes through + /// their file system and serves their branch, normalized here; both managers are built + /// over the two, so no manager is ever left reading through a backend its table moved + /// off. + static std::shared_ptr Create(const std::string& root_path, + CoreOptions core_options); + + const std::string& CurrentBranch() const { + return branch_; + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`. + const std::shared_ptr& GetSchemaManager() const { + return schema_manager_; + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: see + /// `NewSnapshotManager`. + const std::shared_ptr& GetSnapshotManager() const { + return snapshot_manager_; + } + + /// The file system the table's data is accessed through: the one the options resolved, which + /// is the one the caller opened the table through when it opened it through one of its own. + const std::shared_ptr& GetFileSystem() const { + return file_system_; + } + + const std::string& GetRootPath() const { + return root_path_; + } + + /// The final options the table serves, as `Create` took them. Every operator reads them; a + /// copy or a branch switch resolves its own, from the inputs the table keeps. + const CoreOptions& GetOptions() const { + return core_options_; + } + + /// Builds the path factory an operator names its metadata paths through, over this store's + /// root path. The caller names the partition keys of the schema the arrow schema was converted + /// from: an operator serving the table's own schema names this store's partition keys, while + /// a read of a fallback branch names the keys of that branch's schema, which is not this + /// store's. The caller names the file format identifier: a write names the format of its + /// level-0 files, which the options may set per level, while a scan, a read and a commit name + /// the table's format. + Result> NewPathFactory( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const; + + /// Builds the snapshot manager a write or a commit reads snapshots through, over this store's + /// root path and branch. The caller names the file system, from the options its operator + /// serves. A catalog that version-manages the table hands out its own snapshots, so its loader + /// is attached and the catalog is captured to keep it alive. Such a catalog names the table by + /// an identifier, which the context must have supplied. + Result> NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const; + + /// Builds the manifest list a scan or a commit reads the table's manifest entries through. + Result> NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// Builds the manifest file a scan or a commit reads a single manifest's entries through, + /// pruned by `partition_schema`. + Result> NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const; + + /// Builds the index manifest file a write, a scan or a commit reads the table's index + /// manifest through. + Result> NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// Builds the index file handler a write maintains its deletion vectors and indexes through, + /// or a scan reads them through. + Result> NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// Builds the partition computer a commit names the partitions it writes with, over the + /// caller's partition keys and arrow schema. + Result> NewPartitionComputer( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::shared_ptr& memory_pool) const; + + private: + FileStore(const std::string& root_path, CoreOptions core_options); + + std::shared_ptr file_system_; + std::string root_path_; + std::string branch_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; + /// The final options the table serves, as `Create` took them. + CoreOptions core_options_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp new file mode 100644 index 000000000..6e306a2c8 --- /dev/null +++ b/src/paimon/core/table/file_store_table.cpp @@ -0,0 +1,617 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), + store_(store) {} + +Result> FileStoreTable::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::map& fs_scheme_map, + const std::map& dynamic_options) { + std::shared_ptr table_schema; + if (schema != nullptr) { + table_schema = std::dynamic_pointer_cast(schema); + if (table_schema == nullptr) { + return Status::Invalid( + fmt::format("{} is not a managed table schema", identifier.ToString())); + } + } + // Resolve the final options once, before the store is built: the schema's own options as the + // branch restyles them - the main branch carries no `branch` option, another branch names + // itself. A caller with no schema resolves the defaults over its branch, and the operators + // report the missing schema before they ever serve its options. + std::map options = + table_schema != nullptr ? table_schema->Options() : std::map{}; + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, fs_scheme_map)); + std::shared_ptr store = FileStore::Create(root_path, std::move(core_options)); + std::shared_ptr table; + if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { + table = std::make_shared(store, table_schema, identifier, uuid); + } else { + table = std::make_shared(store, table_schema, identifier, uuid); + } + table->dynamic_options_ = dynamic_options; + table->specified_file_system_ = file_system; + table->fs_scheme_map_ = fs_scheme_map; + return table; +} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + } + std::map overridable = dynamic_options; + overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + loaded_schema->MergeOptions(overridable)); + std::map accumulated = dynamic_options_; + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); + } + // A copy resolves its file system from its own merged options, so it takes over no manager of + // the table it was copied from: the store builds both over the resolution the copy names. + return FileStoreTable::Create( + specified_file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, accumulated); +} + +Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + return FileStoreTable::Create(specified_file_system_, root_path_, schema_, identifier_, + CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, + dynamic_options_); + } + // The branch is read through the file system this table serves, and then served through the + // one its own schema resolves: the store builds the managers over that resolution, so none of + // this table's is carried over. + SchemaManager schema_manager(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + std::map overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, + (*latest)->MergeOptions(overridable)); + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); + return FileStoreTable::Create(specified_file_system_, root_path_, + std::shared_ptr(std::move(branch_schema)), + branch_identifier, CatalogUuid().value_or(""), target_branch, + fs_scheme_map_, /*dynamic_options=*/{}); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +Status FileStoreTable::CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + } + return Status::OK(); +} + +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); +} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + std::shared_ptr schema = std::static_pointer_cast(schema_); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->GetOptions(); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); + if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { + return Status::NotImplemented( + "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_path_factory, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + std::shared_ptr io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); +} + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + if (table_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(bool read_optimized, + OptionsUtils::GetValueFromMap(context->GetOptions(), + kReadOptimizedScanOption, false)); + CoreOptions core_options = store_->GetOptions(); + core_options.WithCache(context->GetCache()); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); + auto snapshot_reader = std::make_shared( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + root_path_, snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +Result> FileStoreTable::CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + if (!internal_context->GetIndexScoreBindings().empty()) { + return Status::NotImplemented("Per-field index score output is not implemented yet"); + } + const auto& core_options = internal_context->GetCoreOptions(); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::CreateForTable(read_context, loaded_schema, + store_->GetOptions(), root_path_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + + std::optional scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->GetOptions(); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && + ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == + ctx->GetOptions().end()) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, schema->PartitionKeys(), + options, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_manifest_file, + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); + + auto expire_snapshots = std::make_shared( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, options, ctx->GetTableId(), + ctx->GetCommitUser(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); + + return std::make_unique( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..6c775d65a --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; +class Schema; +class SplitGenerator; + +/// The managed table a catalog hands out: one with snapshots and manifests, described by the +/// schema under its root path. A format table - a directory of plain data files - is a +/// `FormatTable`, the other kind of `Table`. +/// +/// The four operators share one flow, the one Java Paimon's `AbstractFileStoreTable` carries. What +/// the table was loaded with - schema, file system, root path, branch and managers - decides what +/// every operator serves, so it is read once here and handed down. The schema's primary keys decide +/// the kind of managed table it is, `AppendOnlyFileStoreTable` or `PrimaryKeyFileStoreTable`, which +/// the `Create` factory below picks. That kind then builds the pieces the flow hands down: the +/// write itself, the scans, the read and the split generator. +class FileStoreTable : public Table { + public: + /// Builds the managed table the schema describes, as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. + /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can + /// neither add nor drop primary keys, so every schema the root path ever held names one kind. + /// The final options are resolved here once, before the store is built: the schema's own + /// options as `branch` restyles them - the main branch carries no `branch` option, another + /// branch names itself - are parsed, and a parse failure is this call's error. Every operator + /// and both managers are built over that one resolution. + /// @param file_system The file system the caller opened the table through. It wins over the + /// access configuration in the schema's options. Null leaves the resolution to that + /// configuration, and to the default local file system when the options name none. Kept + /// so `Copy` and `SwitchToBranch` resolve through it again. + /// @param schema The schema the table was loaded with. Its primary keys name the kind and its + /// options are what the operators serve. Null when the caller had none, in which case + /// the table is built as the append-only kind and its operators report that none was + /// found. + /// @param branch The branch the table serves, which the managers below read and write on. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through, which the options resolve through as well. Kept so `Copy` and + /// `SwitchToBranch` resolve through it again. + /// @param dynamic_options The options a copy was given over the ones its schema stored, which + /// `Copy` accumulates. See the member of the same name. + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map, + const std::map& dynamic_options); + + const std::string& CurrentBranch() const { + return store_->CurrentBranch(); + } + + const std::shared_ptr& GetSchemaManager() const { + return store_->GetSchemaManager(); + } + + const std::shared_ptr& GetSnapshotManager() const { + return store_->GetSnapshotManager(); + } + + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + + /// Returns a copy that serves `dynamic_options`. The root path and the identifier stay as they + /// are: they name this table, and an option does not rename it. The copy resolves the file + /// system the caller opened this table through, if any, together with its own merged options, + /// and builds its managers over that resolution rather than taking this table's over. + Result> Copy( + const std::map& dynamic_options) const override; + + /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched + /// to is served as it is: its schema is read from its own directory and its managers are built + /// over the file system that schema resolves, so neither this table's dynamic options nor its + /// managers are carried over. Switching to the branch this table already serves hands back a + /// copy of it with the schema shared and the managers built over the same resolution again. A + /// branch with no schema does not exist. + Result> SwitchToBranch(const std::string& branch) const; + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + private: + /// Checks the options a context carries against the ones the table was loaded with. A table + /// serves the options it was loaded with, so an option the context contradicts is refused: a + /// caller that means to change them copies the table. A table a load hands back carries the + /// context's own options merged in - `FileStoreTableFactory` merges them - so it agrees by + /// construction. The refusal belongs to a table the caller already held, whose load is over. A + /// key the table was not loaded with is no contradiction: it names a runtime setting, a scan + /// mode or a batch size, which is none of the table's business and is served as given. + static Status CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options); + + /// Checks the identity a context carries - the path it names, the identifier it gives a + /// catalog, the table id it records - against this table's own. The store below serves this + /// table's files, so a context naming another table would write this table's metadata and + /// publish it as the other's; a field that contradicts the table is refused by name, the way + /// `CheckOptionsAgreeWithTable` refuses a contradicting option. A field the context does not + /// carry is no contradiction: a write or a commit from a bare path names no catalog object. + /// An identifier that names a branch this table does not serve names a branch of another + /// table, as the operator serves the branch the table was loaded with alone. + /// @param kind The operator the context was handed to, which the error names. + /// @param context_table_id The id `CommitContext` records for a catalog commit; a write + /// context carries none, so `NewWrite` passes null. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The + /// path factory comes from the store. The reader is picked by the schema the read serves - the + /// primary keys it carries - not by the table's own kind. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + /// The base constructor the kinds of managed table build through. The store carries the + /// storage they hand down; `schema` is the schema the table was loaded with, which the `Table` + /// base holds and the operators read - null when the caller had none. See `Create` for what + /// each piece is. + FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid); + + /// Restores the committed progress of a real-time write from the latest snapshot the snapshot + /// manager reads. It tells the write where to pick up, and both kinds ask for it the same way. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + /// Returns the scan restricted to the table's root path, which the kinds of file store scan + /// read from. + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// Builds the write of this kind of table. `NewWrite` hands the pieces of the shared flow to + /// it. The schema and the branch the write serves are the table's own. + /// @param io_manager The spill manager the context's temp directory names, or null when the + /// context named none. + /// @param ignore_previous_files Whether the write starts over rather than appends, which the + /// flow resolved with this kind's own rules. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + /// Builds the scan of this kind of table. `NewScan` hands it the manifest list and the manifest + /// file it built, over the table's own schema and managers. + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + /// Builds the split generator of this kind of table, Java Paimon's `splitGenerator()`. + virtual Result> NewSplitGenerator( + const CoreOptions& core_options) const = 0; + + /// Builds the manifest entry scan supplier of this kind of table. `NewCommit` hands it the + /// manifest list and the manifest file it built. + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + + /// The storage this table reads and writes through. See `FileStore`. + std::shared_ptr store_; + + /// The dynamic options the table was loaded with, over the ones its schema stored. `Copy` + /// accumulates them into the copy it derives. They stay separate from the schema they were + /// merged into because a fallback reader needs them on their own: it serves another branch's + /// schema and merges the same options over it again. + std::map dynamic_options_; + + /// The file system the caller opened this table through, as the options were resolved with - + /// null when the load had none. A copy and a branch switch resolve their own options, and the + /// resolution that built this table's store is one-way: neither the file system it produced + /// nor the options hand the inputs back, so they are kept here for the next resolution. + std::shared_ptr specified_file_system_; + + /// The scheme-to-identifier map the options were resolved with, kept for the same next + /// resolution `specified_file_system_` is kept for. + std::map fs_scheme_map_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp new file mode 100644 index 000000000..6231b791b --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid, const std::map& fs_scheme_map) { + SchemaManager schema_manager(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + if (latest_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(latest_schema->Options())); + if (is_format_table) { + return Status::Invalid(fmt::format( + "the schema under {} names a format table, which is not loaded from a path: a " + "catalog hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + path)); + } + } + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // Fold the context options into the schema before the store resolves them: resolving the + // schema's own options first would fail on a format plugin the context overrides away, + // before the override has a chance to take effect. Only options that differ from the + // schema's own restyle the load, the way `Copy` decides; the rest leave the schema and the + // dynamic options as they are. + std::shared_ptr loaded_schema = latest_schema; + std::map dynamic_options; + if (latest_schema != nullptr && !options.empty()) { + const std::map& schema_options = latest_schema->Options(); + bool options_differ = false; + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + options_differ = true; + break; + } + } + if (options_differ) { + dynamic_options = options; + dynamic_options.erase(Options::TYPE); + dynamic_options.erase(Options::BRANCH); + PAIMON_RETURN_NOT_OK( + SchemaValidation::CheckImmutableOptions(*latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + latest_schema->MergeOptions(dynamic_options)); + loaded_schema = std::shared_ptr(std::move(merged_schema)); + } + } + // The schema was read through a file system the context's options resolved, while the table is + // served through the one the schema's own options resolve: the store builds its managers over + // that resolution rather than taking over the reader's. + return FileStoreTable::Create(explicit_file_system, path, loaded_schema, table_identifier, uuid, + branch, fs_scheme_map, dynamic_options); +} + +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); +} + +template +Result> ManagedTable(const Context& ctx, const char* kind) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + ctx.GetPath())); + } + // A schema the caller gave replaces the catalog's; the caller answers for its correctness. + std::optional catalog_table_schema = ctx.GetSpecificTableSchema(); + if (!catalog_table_schema) { + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), ctx.GetBranch())); + PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, + ctx.GetBranch(), kind)); + } + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + ctx.GetBranch(), ctx.GetOptions(), catalog_table_schema, + ctx.GetIdentifier(), ctx.GetTableId().value_or(""), + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +} // namespace + +Result> FileStoreTableFactory::Create(const ScanContext& ctx) { + return ManagedTable(ctx, "scan"); +} + +Result> FileStoreTableFactory::Create(const ReadContext& ctx) { + return ManagedTable(ctx, "read"); +} + +Result> FileStoreTableFactory::Create(const WriteContext& ctx) { + return ManagedTable(ctx, "write"); +} + +Result> FileStoreTableFactory::Create(const CommitContext& ctx) { + return ManagedTable(ctx, "commit"); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h new file mode 100644 index 000000000..e1090dc97 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" + +namespace paimon { + +class CommitContext; +class ReadContext; +class ScanContext; +class Table; +class WriteContext; + +/// Loads the data table a context names, as Java Paimon's `FileStoreTableFactory` does: the schema +/// under the context's path describes it. A system table behind a `T$system` path and a format +/// table are refused - a catalog hands those back through `GetTable`, and their operators come from +/// the table itself - so the static `Create` of all four operators answers the one question this +/// place asks and they cannot disagree. +class FileStoreTableFactory { + public: + FileStoreTableFactory() = delete; + ~FileStoreTableFactory() = delete; + + static Result> Create(const ScanContext& context); + + static Result> Create(const ReadContext& context); + + static Result> Create(const WriteContext& context); + + static Result> Create(const CommitContext& context); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..846842cb7 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -18,9 +18,15 @@ #include "paimon/table/format/format_table.h" +#include +#include #include +#include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +35,22 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +65,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier), options_(options), format_(format), file_compression_(file_compression), @@ -177,35 +194,159 @@ Result> FormatTable::Create( core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata)); } -Result> FormatTable::Copy( - const std::shared_ptr& table, - const std::map& dynamic_options) { - if (table == nullptr) { - return Status::Invalid("format table requires a table to copy"); - } +Result> FormatTable::CopySelf( + const std::map& dynamic_options) const { if (dynamic_options.empty()) { - return table; + return SharedSelf(); } // The options this table already carries go back in as dynamic ones, so an option an earlier // call set is not lost to the schema's own value underneath it. - std::map options = table->options_; + std::map options = options_; for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, - table->location_carries_paimon_metadata_, options); + return Create(file_system_, root_path_, identifier_, + std::static_pointer_cast(schema_), location_carries_paimon_metadata_, + options); } -const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); +Result> FormatTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, CopySelf(dynamic_options)); + return std::shared_ptr
(std::move(copied)); } -std::string FormatTable::FullName() const { - return identifier_.GetFullName(); +const std::vector& FormatTable::PartitionKeys() const { + return LatestSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +Result> FormatTable::NewScan( + std::unique_ptr context) const { + std::shared_ptr shared_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + CopySelf(shared_context->GetOptions())); + if (shared_context->IsStreamingMode()) { + return Status::Invalid( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (shared_context->GetRealtimeContext() != nullptr) { + return Status::Invalid( + "a format table has no real-time store to union with what is on disk"); + } + if (shared_context->GetGlobalIndexResult() != nullptr) { + return Status::Invalid("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = shared_context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::Invalid("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::Invalid("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, shared_context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> FormatTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + if (read_context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + CopySelf(read_context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, read_context)); + return std::unique_ptr(std::move(read)); +} + +Result> FormatTable::NewWrite( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; a format table takes a WriteContextBuilder " + "built from its location instead"); + } + if (context->IsStreamingMode()) { + return Status::Invalid( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::Invalid("a format table has no real-time store to write into"); + } + if (!context->GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + if (context->GetWriteId().has_value()) { + return Status::Invalid("a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> FormatTable::NewCommit( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + CopySelf(context->GetOptions())); + if (!context->IgnoreEmptyCommit()) { + return Status::Invalid( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (context->UseRESTCatalogCommit()) { + return Status::Invalid( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (context->GetTableId()) { + return Status::Invalid( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (context->AppendCommitCheckConflict()) { + return Status::Invalid( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(given_table)); + return std::unique_ptr(std::move(format_commit)); +} + } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp deleted file mode 100644 index 797bc3351..000000000 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "paimon/core/table/format/format_table_loader.h" - -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/identifier.h" -#include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/options/table_type.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace paimon { - -Result> FormatTableLoader::TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, const SchemaManager* schema_manager, - std::shared_ptr* table_schema_out) { - assert(table_schema_out != nullptr); - table_schema_out->reset(); - - std::shared_ptr table_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - // Through the caller's manager when it has one, so that the read warms the cache it goes - // on to use rather than a cache that dies with this call. - SchemaManager own_schema_manager(file_system, table_path, branch); - const SchemaManager& reader = - schema_manager != nullptr ? *schema_manager : own_schema_manager; - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - reader.Latest()); - if (!latest_schema) { - return std::shared_ptr(); - } - table_schema = *latest_schema; - } - *table_schema_out = table_schema; - - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h deleted file mode 100644 index 290ee94a7..000000000 --- a/src/paimon/core/table/format/format_table_loader.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/result.h" - -namespace paimon { - -class FileSystem; -class FormatTable; -class SchemaManager; -class TableSchema; - -/// Recognises a format table behind a table path. -/// -/// The scan, read, write and commit entry points all take a path and have to answer the same -/// question before they can dispatch, so one place answers it and they cannot disagree. -class FormatTableLoader { - public: - FormatTableLoader() = delete; - ~FormatTableLoader() = delete; - - /// Loads `table_path` as a format table, or returns null when it is not one. - /// - /// Null is also the answer when there is no schema under the path at all: what to say about a - /// table that is not there is the managed path's to decide, and it says it in more detail. - /// - /// The table type is read from the schema, never from `options`: `type` is structural, and one - /// call's options must not decide what kind of table this is. - /// - /// @param branch Branch the schema is read from. Take it from the same place the managed path - /// in the same entry point takes it, or the two dispatch on different schemas. Anything - /// but the main branch is refused once the schema turns out to be a format table's, - /// which has no branches to read. - /// @param options Options given at the call, which win over the ones the schema stored. - /// @param specific_table_schema A schema the caller already holds, as json, which saves - /// reading one from under the path. Used only on the main branch, as on the managed - /// read path. - /// @param schema_manager The manager to read the schema through, or null to use one of this - /// call's own. A caller that has one must pass it: a manager caches what it read. - /// @param table_schema_out Set to the schema this read, or null when there is none, so that a - /// caller falling back to the managed path does not read it twice. - static Result> TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, - const SchemaManager* schema_manager, std::shared_ptr* table_schema_out); -}; - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index e8391f68c..c7456d005 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -58,6 +59,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -761,14 +763,10 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every generic - // entry point merges what the call gave it over what the schema stored, so each runs the same - // checks over the merged result; otherwise a caller could set through a context what the - // schema would have rejected and have it silently dropped. +TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; @@ -776,24 +774,22 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "metastore.partitioned-table"); ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "metastore.partitioned-table"); WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "metastore.partitioned-table"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "metastore.partitioned-table"); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2665,19 +2661,16 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { ASSERT_TRUE(scan.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { - // A caller holding a table path uses the interfaces it uses for every other table. Each one - // recognises a format table from its schema and dispatches to it, the way Java Paimon serves - // both kinds through one `ReadBuilder` and one `BatchWriteBuilder`. +TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(batch))); @@ -2694,26 +2687,26 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { WriteContextBuilder write_id_builder(dir->Str(), "test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_context)), "a write id would name nothing"); CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(empty_commit_context)), "cannot record an empty commit"); CommitContextBuilder rest_builder(dir->Str(), "test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_context)), "rest catalog"); CommitContextBuilder table_id_builder(dir->Str(), "test-user"); table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(table_id_context)), "a format table sends no commit table request"); auto format_catalog = std::make_shared(); @@ -2721,20 +2714,20 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_context)), "keeps no snapshot for a catalog to take"); CommitContextBuilder conflict_builder(dir->Str(), "test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(conflict_context)), "no manifests to check a concurrent commit against"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); @@ -2743,15 +2736,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2764,11 +2755,32 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { reader->Close(); } -TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { - // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its - // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table is handed to the context instead, which - // is what `FormatTable.newReadBuilder()` and `newBatchWriteBuilder()` do in Java. +TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->LatestSchema()); + Identifier identifier("db", "fmt"); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + Result> write = table->NewWrite(std::move(write_context)); + ASSERT_FALSE(write.ok()); + ASSERT_TRUE(write.status().IsInvalid()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + Result> commit = table->NewCommit(std::move(commit_context)); + ASSERT_FALSE(commit.ok()); + ASSERT_TRUE(commit.status().IsInvalid()); +} + +TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2795,11 +2807,11 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // Options given at the call still win over the ones the schema stored, so a file is rolled // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); + WriteContextBuilder write_builder(external->Location(), "test-user"); write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + external->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2809,25 +2821,25 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + CommitContextBuilder commit_builder(external->Location(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + external->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + ScanContextBuilder scan_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + external->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + ReadContextBuilder read_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + external->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2843,70 +2855,169 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. +TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr
table; + ASSERT_OK_AND_ASSIGN(table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(read_batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + ImportBatch(read_batch)); + auto ids = checked_pointer_cast(record_batch->column(1)); + auto names = checked_pointer_cast(record_batch->column(2)); + auto dts = checked_pointer_cast(record_batch->column(3)); + for (int64_t i = 0; i < record_batch->num_rows(); i++) { + rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" + + dts->GetString(i)); + } + } + reader->Close(); + ASSERT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); +} + +TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + std::shared_ptr
table = format_table; + + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); + + auto streaming_scan_context = [&dir]() -> Result> { + ScanContextBuilder builder(dir->Str()); + builder.WithStreamingMode(true); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, + streaming_scan_context()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), + "nothing for a streaming scan to follow"); + + auto vector_read_context = [&dir]() -> Result> { + ReadContextBuilder builder(dir->Str()); + builder.SetVectorSearch(std::make_shared( + /*field_name=*/"name", /*limit=*/1, std::vector{0.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, /*distance_type=*/std::nullopt, + /*options=*/std::map{})); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), + "not supported for format tables"); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(format_table->LatestSchema()); + + auto catalog_write_context = [&dir, &catalog]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, + catalog_write_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), + "WithCatalog() requires a native table"); + + auto catalog_commit_context = [&dir, &catalog]() -> Result> { + CommitContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, + catalog_commit_context()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), + "keeps no snapshot for a catalog to take"); + + auto write_id_context = [&dir]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + builder.WithWriteId(3); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), + "a write id would name nothing"); +} + +TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a format table"); - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); + ScanContextBuilder served_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, + table->NewScan(std::move(served_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, served_scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 0u); +} - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); - - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); - - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); -} - -TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { - // `WithBranch()` is refused on a context built from a format table, and the `branch` option - // says the same thing by another name: a format table keeps no metadata to branch, and a read - // or a write would go to its one location whatever the option named. Refusing it is the only - // answer that is not silently the wrong one - an overwrite would delete the main branch's - // data while claiming to replace another branch's. +TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2916,32 +3027,32 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK(WriteAndCommit(table, std::move(batch))); const std::map branch = {{Options::BRANCH, "dev"}}; - ASSERT_NOK_WITH_MSG(FormatTable::Copy(table, branch), "has no branches"); + ASSERT_NOK_WITH_MSG(table->Copy(branch), "has no branches"); ASSERT_NOK_WITH_MSG( FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); + ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "has no branches"); - ReadContextBuilder read_builder(table); + ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "has no branches"); - WriteContextBuilder write_builder(table); + WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "has no branches"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "has no branches"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, @@ -2953,15 +3064,29 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) // The branch that names no branch at all is the main one, and is left alone: refusing it // would refuse the default every context carries. for (const char* main_branch : {"main", ""}) { - ASSERT_OK(FormatTable::Copy(table, {{Options::BRANCH, main_branch}})) << main_branch; + ASSERT_OK(table->Copy({{Options::BRANCH, main_branch}})) << main_branch; } } +TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); + std::shared_ptr copied_table = std::static_pointer_cast(copied); + EXPECT_EQ(copied_table->Options().at("probe"), "on-top"); + EXPECT_EQ(copied_table->Options().at(Options::TYPE), "format-table"); + EXPECT_EQ(copied_table->FullName(), table->FullName()); + EXPECT_EQ(copied_table->Location(), table->Location()); + EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); + + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); +} + TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { - // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch is what the entry point dispatches on. A format table's data is the files under the - // table path whichever branch named it, so serving one from there under another branch's name - // would answer a question about `dev` with the rows of `main`. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2980,29 +3105,25 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr ReadContextBuilder read_builder(dir->Str()); read_builder.WithBranch("dev"); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions({{Options::BRANCH, "dev"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - // The main branch is untouched by any of it. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + table->NewScan(std::move(main_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); } -TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { - // A context promises that options given at the call win over the ones the schema stored, and - // that a branch and a caller-held schema are honoured. Dispatching to a format table must not - // quietly drop any of that. +TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3014,7 +3135,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3024,41 +3145,19 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the generic commit, or nothing below would see the rows: until the commit - // renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + table->NewCommit(std::move(generic_commit_context))); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on is a managed table as far as dispatch is concerned, so - // the format branch must not answer for it. ReadContextBuilder branch_builder(dir->Str()); branch_builder.WithBranch("nosuchbranch"); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_NOK(TableRead::Create(std::move(branch_context))); - // A caller-held schema is used instead of reading one from under the path. - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); - // Planning, not just dispatch: handing the schema over must not turn the `schema` directory - // the table keeps under its own path into data, which a plan would then read as a data file. - ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::vector seeded_rows, - ReadAll(table, seeded_plan->Splits())); - ASSERT_EQ(seeded_rows.size(), 3u); - // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. @@ -3066,7 +3165,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { retyped_builder.SetOptions({{Options::TYPE, "table"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + table->NewScan(std::move(retyped_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3101,7 +3200,7 @@ class CountingCache : public Cache { } // namespace -TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { +TEST(FormatTableTest, TestTheReadCarriesPrefetchAndTheCacheThrough) { // A format table opens its files through the same component the managed table path opens its // own with, so what a `ReadContext` asks for about opening a file applies here too. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3115,13 +3214,11 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3156,7 +3253,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ASSERT_GT(cache->gets, 0) << "the cache the read context carries never reached the file reader"; } -TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { +TEST(FormatTableTest, TestTheOperatorsRefuseWhatAFormatTableCannotHonour) { // Anything a context carries that a format table cannot do is a refusal naming the setting, // never a read or a write that quietly did something else. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3171,14 +3268,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_schema_context)), "does not take a projected read schema"); ScanContextBuilder streaming_builder(dir->Str()); streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context)), "nothing for a streaming scan to follow"); ScanContextBuilder predicate_builder(dir->Str()); @@ -3186,13 +3283,12 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), - "does not take a predicate"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(predicate_context)), "does not take a predicate"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3213,12 +3309,12 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { // types then finds out at the call rather than from a table that did not change. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3262,7 +3358,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3275,7 +3371,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..355ebf148 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template +class MergeFunctionWrapper; + +namespace { + +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, schema, identifier, uuid) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + std::shared_ptr schema = std::static_pointer_cast(schema_); + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), ctx.GetFileSystemSchemeToIdentifierMap(), + ctx.GetExecutor(), ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE(index_file_handler, store_->NewIndexFileHandler( + options, path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, CurrentBranch(), schema, primary_key_index_definitions.Definitions(), + path_factory, index_file_handler, options, io_manager, ctx.EnableMultiThreadSpill(), + ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..121081735 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema with primary keys describes, Java Paimon's +/// `PrimaryKeyFileStoreTable`. Its write merges by key, and its scan, read and commit read +/// through the merge. The operator flows are the ones `FileStoreTable` carries; the pieces +/// those flows hand down are built here. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds a primary key table through: the store + /// carries the storage state, `schema` is the schema the table was loaded with, and the base + /// class holds both. + PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp index 42a7171b3..cfb61c17d 100644 --- a/src/paimon/core/table/source/realtime_table_scan.cpp +++ b/src/paimon/core/table/source/realtime_table_scan.cpp @@ -27,17 +27,21 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/plan_impl.h" #include "paimon/core/table/source/realtime_split.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/realtime/realtime_context.h" #include "paimon/realtime/realtime_store.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { @@ -203,6 +207,49 @@ Result>> RealtimeTableScan::CreateRealtimeSpl return result; } +Status RealtimeTableScan::ValidateScan(const TableSchema& table_schema, + const CoreOptions& core_options, const ScanContext& context, + bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + RealtimeTableScan::RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 821e67fd7..318319139 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -30,14 +30,23 @@ namespace paimon { +class CoreOptions; class FileStorePathFactory; class FileSystem; +class ScanContext; class ScanFilter; class SnapshotManager; +class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: + /// The checks a scan that asks for a real-time union read opens with, run before one of these + /// is built: the table has to be able to serve it, and the options have to name a scan one + /// can be served over. A context that asks for none of it passes straight through. + static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized); + RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index e723c54bb..5688c7650 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,8 +19,18 @@ #include "paimon/table/source/table_read.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + namespace paimon { +Result> TableRead::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewRead(std::move(ctx)); +} + Result> TableRead::CreateCountReader( const std::vector>& splits) { (void)splits; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..c7724529b 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -134,20 +135,24 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, table_path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -159,13 +164,17 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -250,25 +259,41 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - auto append_only_table_read = dynamic_cast(table_read.get()); - ASSERT_TRUE(append_only_table_read); - auto* typed_split_read = - dynamic_cast(append_only_table_read->split_reads_[0].get()); - ASSERT_TRUE(typed_split_read); - const auto& core_options = typed_split_read->options_; - std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; - ASSERT_EQ(expected_options, core_options.ToMap()); + { + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("read.batch-size", "2"); + context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + const auto& core_options = typed_split_read->options_; + std::map expected_options = { + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "2"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}}; + ASSERT_EQ(expected_options, core_options.ToMap()); + } + { + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("bucket", "10"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + ASSERT_EQ(10, typed_split_read->options_.GetBucket()); + } } TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { @@ -300,7 +325,7 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index be5d5b955..d3dd794b4 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,10 +19,20 @@ #include "paimon/table/source/table_scan.h" +#include "paimon/catalog/table.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/scan_context.h" namespace paimon { +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); + return table->NewScan(std::move(context)); +} + std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..4fd0f9873 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -19,12 +19,15 @@ #include "paimon/table/source/table_scan.h" +#include #include #include #include #include #include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -164,12 +167,17 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; + std::map options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), path, options)); ScanContextBuilder builder(path); builder.AddOption(Options::FILE_FORMAT, "orc"); builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(context)), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..2c3adec8c 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -44,6 +44,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -358,15 +359,19 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : fs_(std::move(fs)), + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -383,17 +388,24 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options_) .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(data_table_fs) + .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -411,7 +423,10 @@ Result> AuditLogSystemTable::NewScan( } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -450,12 +465,16 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(read_options) .SetReadFieldNames(base_read_schema->field_names()) .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..4cc324b1d 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -58,14 +58,17 @@ class AuditLogSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: + /// For a derived changelog view that answers with a kind of its own: `BinlogSystemTable` names + /// itself `T$binlog`, not the `T$audit_log` the public constructor composes. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..8299f17f5 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -73,12 +74,8 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = @@ -98,7 +95,10 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..3a28ffa93 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -40,10 +40,8 @@ class BinlogSystemTable : public AuditLogSystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..e420125a7 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,6 +126,16 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } +bool RequiresCatalog(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.requires_catalog; + } + } + return false; +} + struct CatalogTableInfo { std::string database_name; std::string table_name; @@ -134,11 +144,15 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +Result> LoadAllDataTables( + const std::shared_ptr& catalog, const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +161,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -288,6 +302,14 @@ Result GlobalSystemTableLoader::IsSupported( } Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + if (context.catalog == nullptr && RequiresCatalog(table_name)) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } + return LoadForSchema(table_name, context); +} + +Result> GlobalSystemTableLoader::LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -296,11 +318,13 @@ Result> GlobalSystemTableLoader::Load( if (!enabled) { return Status::NotExist("global system table is disabled: ", table_name); } - if (entry.requires_catalog && context.catalog == nullptr) { - return Status::NotImplemented("global system table requires catalog context: ", - table_name); - } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + entry.factory(context)); + // TODO(yonghao.fyh): the loader-driven AdoptOwnSchema() is an init-style coupling; + // consider overriding LatestSchema() on SystemTable, as FormatTable does, to serve + // its own read-only schema and drop this call from every loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,11 +348,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -360,11 +380,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -380,7 +397,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -399,11 +416,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -429,7 +442,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -471,11 +484,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -495,7 +504,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..280f57479 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,11 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// The catalog this table reads its rows through. Rows are built lazily, so a table handed back + /// to a caller keeps shared ownership of it. Empty when the table is transient and only its + /// schema is read. `GlobalSystemTableLoader::Load` refuses a table that enumerates data tables + /// when this is empty. + std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -48,7 +52,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -64,7 +67,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -80,7 +82,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -96,7 +97,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -113,9 +113,17 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table for a caller to read. A table that enumerates data tables needs + /// the catalog in the context, so it is refused here when the context carries none. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); + /// Loads a global system table for the schema it exposes alone, the way + /// `Catalog::LoadTableSchema` does. No rows are built, so a context that carries no catalog is + /// accepted here and refused by `Load` instead. + static Result> LoadForSchema( + const std::string& table_name, const GlobalSystemTableContext& context); + static Result> GetSupportedTableNames( const std::map& catalog_options); }; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..a2c69195a 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,8 +29,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,16 +106,20 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { + std::unique_ptr /*context*/) const { return std::make_unique(table_path_); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..c72d7389e 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -28,18 +28,20 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5279e6871..1b5927004 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -504,13 +504,11 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -532,13 +530,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -594,13 +588,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -650,13 +640,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -700,13 +686,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -739,13 +721,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -774,14 +752,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -843,14 +817,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..1127e37ed 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,9 +35,9 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -62,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -77,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -92,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -107,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -123,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -140,7 +135,6 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -157,7 +151,6 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..ff6135178 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -27,6 +27,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -35,17 +36,15 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path, ComposeName(table_path, kName)), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -57,18 +56,25 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(data_table_fs) + .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -100,6 +106,10 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithBranch(branch) .SetPredicate(context->GetPredicate()) @@ -111,7 +121,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) @@ -133,7 +143,10 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..82fec3ef4 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,13 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..872f6e502 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -28,6 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/schema_manager.h" @@ -37,8 +38,14 @@ #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { @@ -70,11 +77,11 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& /*dynamic_options*/) -> Result> { - return std::make_shared(table_path, table_schema); + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -93,12 +100,12 @@ const std::vector& SystemTableRegistry() { fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -176,6 +183,49 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& table_path, const std::string& kind) { + return PathUtil::GetName(table_path) + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} + +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch does not support system-table reads yet"); + } + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + return Status::OK(); +} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::Copy( + const std::map& /*dynamic_options*/) const { + return Status::NotImplemented(Name(), " system table is read-only: copying is not supported"); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -186,7 +236,10 @@ Result> SystemTableLoader::Load( const std::map& dynamic_options) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options)); + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } @@ -237,7 +290,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..640672236 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,6 +25,8 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" @@ -33,9 +35,8 @@ namespace paimon { class FileSystem; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +50,64 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// Base class for the table-scoped system tables such as `T$options` and `T$snapshots`. A +/// `T$system` identifier names a read-only view that exposes a schema of its own and serves scans +/// and reads from the data table it belongs to. /// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// A system table of a data table answers `$` for `Name()`. One of the `sys` +/// database answers the bare kind name. `FullName()` returns the same as `Name()` in both cases, +/// which is what Java Paimon's `Table.fullName()` defaults to. +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string Name() const override { + return name_; + } - virtual std::string Name() const = 0; + std::string FullName() const override { + return name_; + } + + /// Returns the read-only schema this system table exposes. It is not the data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + /// A system table is read-only, so copying it, writing it and committing for it are refused + /// instead of routed to the data table underneath. That table serves them through its own + /// `Table`. + Result> Copy( + const std::map& dynamic_options) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs File system the data table underneath is read through. + /// @param table_path Root path of the data table this system table belongs to. + /// @param name The name this table answers with: `ComposeName(table_path, kind)` for a + /// table-scoped system table, the bare kind name for one of the `sys` database. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes the name of a table-scoped system table. Java Paimon's `OptionsTable.name()` + /// composes it the same way: `location.getName() + SYSTEM_TABLE_SPLITTER + OPTIONS`. + static std::string ComposeName(const std::string& table_path, const std::string& kind); + + /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` + /// predicate. Neither is evaluated on the rows a system table serves, so answering one + /// would return plain rows and quietly drop the search and its top K. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Fills in this table's own read-only `SystemTableSchema`. A loader calls it once the object + /// exists, because `ArrowSchema()` is virtual and cannot run from a constructor. + Status AdoptOwnSchema(); + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; + + private: + std::string name_; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..4e8a03516 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,6 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" @@ -38,20 +39,31 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { namespace { @@ -219,7 +231,7 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -227,6 +239,18 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } +TEST(SystemTableTest, TestCopyIsRefused) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_NOK_WITH_MSG(system_table->Copy({{"probe", "on-top"}}), "copying is not supported"); +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); @@ -249,7 +273,7 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); @@ -287,6 +311,106 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(audit_log.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); +} + +TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { + const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; + + ScanContextBuilder scan_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a system table"); + + ReadContextBuilder read_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); + + WriteContextBuilder write_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a system table"); + + CommitContextBuilder commit_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a system table"); +} + +TEST(SystemTableTest, TestNewReadRefusesASearch) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto make_context = [] { + auto vector_search = std::make_shared( + "v", /*limit=*/3, std::vector{1.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, VectorSearch::DistanceType::COSINE, + std::map{}); + ReadContextBuilder builder("/tmp/table"); + builder.SetVectorSearch(vector_search); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(audit_log.NewRead(std::move(audit_log_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(std::move(binlog_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); +} + +TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, + SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + + auto make_context = [&predicate] { + ReadContextBuilder builder("/tmp/table"); + builder.SetPredicate(predicate); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr in_memory_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(in_memory->NewRead(std::move(in_memory_context)), + "FullTextMatch predicate execution is not implemented yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "FullTextMatch predicate execution is not implemented yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one @@ -319,21 +443,23 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); - ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + std::shared_ptr catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), + options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::Schema typed_schema( {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); - Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, - /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, - /*ignore_if_exists=*/false); + Status created = catalog->CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); ArrowSchemaRelease(&c_schema); ASSERT_OK(created); GlobalSystemTableContext context; - context.catalog = &catalog; + context.catalog = catalog; context.fs = core_options.GetFileSystem(); context.warehouse = dir->Str(); context.catalog_options = options; @@ -343,16 +469,16 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { // still the ones asked for before its files are listed. ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); ASSERT_TRUE(rows.empty()); - ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + ASSERT_EQ((std::vector{"db1.t1"}), catalog->requested); // A table whose credentials cannot be issued fails the query, so that a partial result // is never mistaken for the whole one. - catalog.failure = Status::Invalid("no credentials for the table"); + catalog->failure = Status::Invalid("no credentials for the table"); ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); // A table that went away between being listed and being read is skipped instead, like // one whose location has already been removed. - catalog.failure = Status::NotExist("table dropped"); + catalog->failure = Status::NotExist("table dropped"); ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); ASSERT_TRUE(skipped.empty()); } diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index dd7e18f20..3180a48fe 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -9,202 +9,17 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/catalog/table.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/predicate/predicate_validator.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/commit_lock.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" -#include "paimon/core/catalog/snapshot_commit.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/disk/io_manager.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" -#include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" -#include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_commit_impl.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" -#include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" -#include "paimon/core/table/source/data_table_batch_scan.h" -#include "paimon/core/table/source/data_table_stream_scan.h" -#include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" -#include "paimon/core/table/source/primary_key_index_batch_scan.h" -#include "paimon/core/table/source/read_optimized_scan_options.h" -#include "paimon/core/table/source/realtime_table_scan.h" -#include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/file_store_commit.h" -#include "paimon/file_store_write.h" -#include "paimon/format/file_format.h" -#include "paimon/fs/file_system.h" -#include "paimon/predicate/predicate_utils.h" -#include "paimon/read_context.h" -#include "paimon/realtime/realtime_context.h" -#include "paimon/result.h" -#include "paimon/scan_context.h" -#include "paimon/schema/schema.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" -#include "paimon/table/source/table_read.h" -#include "paimon/table/source/table_scan.h" -#include "paimon/write_context.h" - -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { -class DataSplit; -class Executor; -struct KeyValue; -class MemoryPool; -template -class MergeFunctionWrapper; - -namespace { - -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - -} // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -215,1227 +30,4 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::NewScan(std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> Table::NewRead(std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> Table::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> Table::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); -} - -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); - if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { - return Status::NotImplemented( - "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - std::shared_ptr io_manager; - const auto& io_temp_dir = ctx->GetTempDirectory(); - if (!io_temp_dir.empty()) { - io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); - } - - bool ignore_previous_files = ctx->IgnorePreviousFiles(); - if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { - return Status::Invalid("real-time write requires realtime.enabled=true"); - } - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } -} - -namespace { - -class TableScanImpl { - public: - static Result> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); - // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), context); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - std::shared_ptr shared_context = std::move(context); - // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); - } - // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); - } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); -} - -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); - } - PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), - kReadOptimizedScanOption, false)); - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); - core_options.WithCache(context->GetCache()); - - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - - PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); - // validate options - if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); - } - } else if (core_options.GetBucket() < 1 && - !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { - return Status::Invalid( - fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); - } - - // validate schema and scan filter - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { - PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( - *arrow_schema, context->GetScanFilters()->GetPredicate(), - /*validate_field_idx=*/false)); - } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); - auto snapshot_reader = std::make_shared( - file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); - const bool pk_table = !table_schema->PrimaryKeys().empty(); - if (read_optimized && pk_table && context->IsStreamingMode()) { - return Status::NotImplemented( - "read-optimized system table does not support streaming scan for primary key table"); - } - if (context->IsStreamingMode()) { - return std::make_unique(core_options, snapshot_reader); - } - auto batch_scan = std::make_unique( - /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), - /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); - if (context->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, - RealtimeContextImpl::Cast(context->GetRealtimeContext())); - return std::make_unique( - std::move(batch_scan), pk_table, realtime_context, path_factory, - snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), - context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); - } - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, - context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), - context->GetExecutor()); - } - if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, - PrimaryKeyIndexDefinitions::Create(*table_schema)); - if (!definitions.ScalarDefinitions().empty()) { - return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), - table_schema, path_factory, core_options, - context->GetMemoryPool()); - } - } - return batch_scan; -} - -} // namespace - -namespace { - -/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and -/// reading it again would cost a second listing and read. -Result> CreateInternalReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } - return InternalReadContext::Create(context, table_schema, options); -} - -Result> CreateTableRead( - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - if (!internal_context->GetIndexScoreBindings().empty()) { - return Status::NotImplemented("Per-field index score output is not implemented yet"); - } - const auto& core_options = internal_context->GetCoreOptions(); - const auto& table_schema = internal_context->GetTableSchema(); - if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { - return Status::Invalid("real-time read requires realtime.enabled=true"); - } - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); - - std::optional scan_fallback_branch = - internal_context->GetCoreOptions().GetScanFallbackBranch(); - if (!scan_fallback_branch || - StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { - return std::move(table_read); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); - const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); - return std::make_unique(std::move(table_read), - std::move(fallback_table_read), arrow_pool); -} - -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - -} // namespace - -Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - return NewFormatTableRead(format_table, context); - } - - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); -} - -namespace { - -CommitScanner::ScanSupplier CreateAppendScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); - } - if (ctx.UseRESTCatalogCommit()) { - if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); - } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(options.GetFileSystem(), snapshot_manager)); -} - -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - -} // namespace - -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - schema_id_loader = [catalog, identifier]() -> Result { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid( - "the catalog did not hand back a data table schema, which has the id a " - "snapshot records"); - } - return data_schema->Id(); - }; - } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); - } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - std::shared_ptr arrow_schema = - DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - assert(options.GetFileSystem()); - assert(options.GetFileFormat()); - PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { - return Status::NotImplemented( - "commit operation does not support object store file system for now"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - - auto expire_snapshots = std::make_shared( - snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, - options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), - ctx->GetExecutor()); - - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); - - return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), - options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, - ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, - std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); -} - } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..43065e16b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -25,20 +25,36 @@ #include #include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/cache/cache.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -46,24 +62,642 @@ namespace paimon::test { namespace { -// Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier`. -Result> CreateManagedTable(const std::shared_ptr& file_system, - const std::string& path, - const Identifier& identifier, - const std::string& branch = "main") { +Result> CreateManagedTable( + const std::shared_ptr& file_system, const std::string& path, + const Identifier& identifier, const std::string& branch = "main", + const std::map& options = {}) { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( + /*primary_keys=*/{}, options)); + return FileStoreTable::Create( file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); +} + +void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + +void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, + int32_t id, int32_t v) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + arrow::Int32Builder vs_builder; + ASSERT_TRUE(ids_builder.AppendValues({id}).ok()); + ASSERT_TRUE(vs_builder.AppendValues({v}).ok()); + std::shared_ptr ids_array; + std::shared_ptr vs_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + ASSERT_TRUE(vs_builder.Finish(&vs_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array, vs_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); } } // namespace +TEST(TableTest, TestTableCarriesManagersOfItsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, path, Identifier(Identifier::kUnknownDatabase, "tbl"), + /*branch=*/"dev")); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); + // A copy builds its managers over its own resolution rather than taking this table's over, and + // so serves the same branch through managers of its own. + ASSERT_TRUE(file_store_copy->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_copy->GetSnapshotManager() != nullptr); + EXPECT_NE(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Fs(), fs); +} + +TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + file_store_copy->NewScan(std::move(ctx))); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, + branch_table->Copy({{Options::BRANCH, "main"}})); + auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); + ASSERT_TRUE(branch_file_store_copy != nullptr); + EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); + { + ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); + scan_builder.AddOption(Options::BRANCH, "dev"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + branch_file_store_copy->NewScan(std::move(ctx))); + } +} + +TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + off->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto partitioned_schema = + arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32()), arrow::field("dt", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(partitioned_schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, + {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + // Every batch reader leads with `_VALUE_KIND`, ahead of the fields the read named. + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + auto make_table = + [&](const std::string& path, + const std::map& options) -> Result> { + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + if (options.count(Options::SCAN_FALLBACK_BRANCH) != 0) { + SchemaManager fallback_schema_manager(fs, path, "fb"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {})); + } + return FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
with_fallback, + make_table(dir->Str() + "/fallback", {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + + for (const std::shared_ptr
& table : {plain, with_fallback}) { + SCOPED_TRACE(table->GetRootPath()); + ReadContextBuilder read_builder(table->GetRootPath()); + read_builder.AddOption(Options::FILE_FORMAT, "bogus"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(read_context))); + } +} + +TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable( + fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {"s3.endpoint", "http://old"}})); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + + // An option that moves the file system the schema named and one that leaves it where it was + // both come back as a copy of their own: a copy never reads metadata through the managers of + // the table it was copied from, whichever resolution it ends up serving. + for (const std::string& endpoint : {std::string("http://new"), std::string("http://old")}) { + SCOPED_TRACE(endpoint); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{"s3.endpoint", endpoint}})); + auto copy_file_store_table = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(copy_file_store_table != nullptr); + EXPECT_NE(copy_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(copy_file_store_table->GetSnapshotManager(), + file_store_table->GetSnapshotManager()); + EXPECT_EQ("main", copy_file_store_table->CurrentBranch()); + auto copy_schema = std::dynamic_pointer_cast(copy->LatestSchema()); + ASSERT_TRUE(copy_schema != nullptr); + EXPECT_EQ(endpoint, copy_schema->Options().at("s3.endpoint")); + } +} + +TEST(TableTest, TestSwitchToBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + auto main_file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(main_file_store_table != nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, + main_file_store_table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); + ASSERT_TRUE(dev_table_schema != nullptr); + ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, + dev_file_store_table->SwitchToBranch("dev")); + auto same_file_store_table = std::dynamic_pointer_cast(same_table); + ASSERT_TRUE(same_file_store_table != nullptr); + // Switching to the branch a table already serves shares its schema and resolves again: the + // managers it hands back are instances of their own, reading the same path on the same branch. + EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); + EXPECT_NE(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_NE(same_file_store_table->GetSnapshotManager(), + dev_file_store_table->GetSnapshotManager()); + EXPECT_EQ("dev", same_file_store_table->CurrentBranch()); + EXPECT_EQ(dev_table->GetRootPath(), same_table->GetRootPath()); + EXPECT_EQ(path, same_file_store_table->GetSnapshotManager()->RootPath()); + EXPECT_EQ("dev", same_file_store_table->GetSnapshotManager()->Branch()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, + dev_file_store_table->SwitchToBranch("main")); + auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); + ASSERT_TRUE(main_again_file_store_table != nullptr); + EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl", main_again->Name()); + EXPECT_EQ("tbl", main_again->FullName()); + ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); + ASSERT_TRUE(main_again_schema != nullptr); + EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); + + ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); +} + +TEST(TableTest, TestSwitchToBranchFromATableThatHasNoSchemaOfItsOwn) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/tbl"; + SchemaManager dev_schema_manager(dir->GetFileSystem(), path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + // A table opened over a path its own branch holds no schema under, and opened without a file + // system of its own: its operators report that no schema was found, while a branch that has one + // is read through the default file system the store falls back to and served on its own. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, + Identifier("db", "tbl"), /*uuid=*/"", + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); +} + +TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + std::vector branch_entries; + ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); + EXPECT_FALSE(branch_entries.empty()); +} + +TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + EXPECT_FALSE(plan->Splits().empty()); + + ScanContextBuilder main_scan_builder(dir->Str()); + main_scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, + table->NewScan(std::move(main_scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr main_plan, main_scan->CreatePlan()); + EXPECT_FALSE(main_plan->Splits().empty()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -162,11 +796,12 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself calls the constructor directly, which a - // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); - EXPECT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + EXPECT_EQ("main", table->CurrentBranch()); } } @@ -178,15 +813,16 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); - EXPECT_EQ(branch, "main"); + auto main_file_store_table = std::dynamic_pointer_cast(main_table); + ASSERT_TRUE(main_file_store_table != nullptr); + EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The identifier a table was opened with names the branch it is for. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); - EXPECT_EQ(branch, "dev"); + auto branch_file_store_table = std::dynamic_pointer_cast(branch_table); + ASSERT_TRUE(branch_file_store_table != nullptr); + EXPECT_EQ(branch_file_store_table->CurrentBranch(), "dev"); } TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { @@ -199,10 +835,8 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + EXPECT_NE(opened->GetFileSystem(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); @@ -211,10 +845,7 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,7 +861,9 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; @@ -287,6 +920,117 @@ TEST(TableTest, TestNewScanCreatesATableScan) { table->NewScan(std::move(ctx))); } +TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(held_ctx))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -341,6 +1085,89 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { table->NewCommit(std::move(ctx))); } +TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto other_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(other_dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + { + WriteContextBuilder write_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names root path"); + } + { + CommitContextBuilder commit_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names root path"); + } + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "tbl$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names branch"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.UseRESTCatalogCommit(true).WithTableId("another-table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "table id"); + } + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + table->NewCommit(std::move(ctx))); +} + +TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev$x")); + + CommitContextBuilder fs_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs_ctx, fs_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr fs_commit, + table->NewCommit(std::move(fs_ctx))); + + CommitContextBuilder rest_builder(dir->Str(), "commit-user"); + rest_builder.UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_ctx, rest_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_ctx)), "cannot contain"); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_ctx)), "cannot contain"); +} + TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -356,6 +1183,296 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } +TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); + ScanContextBuilder agreeing_scan_builder(dir->Str()); + agreeing_scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, agreeing_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + copied->NewScan(std::move(ctx))); +} + +TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_EQ(plan->SnapshotId(), std::optional(1)); + + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_ctx))); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
copied, + table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + copied->NewScan(std::move(copied_scan_ctx))); + ASSERT_NOK_WITH_MSG(copied_scan->CreatePlan(), "snapshot"); +} + +TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("id", arrow::int32(), + /*nullable=*/false)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); + auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + ASSERT_NOK_WITH_MSG( + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), + "target-file-row-num should be at least 1"); +} + +TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + auto cache = std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(cache->GetCount(CacheKind::MANIFEST), 0) + << "the cache the scan context carries never reached the scan"; +} + +TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copied->LatestSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); + EXPECT_EQ(copied_schema->Id(), + std::dynamic_pointer_cast(table->LatestSchema())->Id()); + EXPECT_EQ(copied->FullName(), table->FullName()); + EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); + EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); + EXPECT_EQ( + std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), + "1"); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); +} + +TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), + "Change 'bucket-function.type' is not supported yet."); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
restyled, + table->Copy({{Options::MERGE_ENGINE, "deduplicate"}})); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET_KEY, "v"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "Change 'bucket-key' is not supported yet."); + } + + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, + table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ignore_delete_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/ignore_delete", + Identifier("db", "igd"), /*branch=*/"main", + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(ignore_delete_table->Copy({{Options::IGNORE_DELETE, "false"}}), + "Cannot change ignore-delete from true to false."); + } +} + +TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + + const std::string table_dir = dir->Str() + "/my_db.db/my_table"; + ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, {{"file.format", "orc"}})); + { + ScanContextBuilder builder(table_dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "my_db.my_table"); + } + + const std::string loose_table_dir = dir->Str() + "/loose_table"; + ASSERT_OK(fs->Mkdirs(loose_table_dir)); + SchemaManager loose_schema_manager(fs, loose_table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr loose_schema, + loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"file.format", "orc"}})); + { + ScanContextBuilder builder(loose_table_dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "loose_table"); + } +} + +TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + // A schema whose stored format this process cannot resolve: resolving the schema's own + // options at load would fail on it, so the load only serves the table when the context + // overrides the format, and only when the override is merged before the resolution. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stored_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {{"file.format", "unavailable-format"}})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, stored_schema->GetJsonSchema()); + { + ScanContextBuilder builder(dir->Str()); + builder.SetTableSchema(schema_json); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK(FileStoreTableFactory::Create(*ctx)); + } + { + ScanContextBuilder builder(dir->Str()); + builder.SetTableSchema(schema_json).AddOption(Options::FILE_FORMAT, "parquet"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_NE(table, nullptr); + // The scan builds its path factory over the format the merged options resolve, so it + // only opens when the override, not the stored format, is what the table serves. + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_NE(scan, nullptr); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -393,4 +1510,65 @@ TEST(TableTest, TestCreateBuildsTheOperator) { FileStoreCommit::Create(std::move(commit_ctx))); } +TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(table->LatestSchema()); + + // A schema the caller gave replaces the catalog's on the main branch, so neither the write + // nor the commit asks the catalog for one. + ASSERT_OK_AND_ASSIGN(std::string schema_json, table->LatestSchema()->GetJsonSchema()); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_ctx, + write_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, + FileStoreWrite::Create(std::move(write_ctx))); + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_ctx, + commit_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_ctx))); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0U); +} + +TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(table->LatestSchema()); + + // All four loads take the schema the catalog holds on the main branch when the caller did + // not set one, so each of them asks the catalog exactly once. + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + scan_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scanned, FileStoreTableFactory::Create(*scan_ctx)); + ASSERT_EQ(scanned->FullName(), "db.tbl"); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, + read_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, + FileStoreTableFactory::Create(*read_ctx)); + ASSERT_EQ(read_table->FullName(), "db.tbl"); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 2U); +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..57e337904 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +namespace paimon { + +/// The checks every kind of table's operator entry points share. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// The checks every operator entry point opens with: a context, a memory pool and an executor + /// must all be there for the operator to be built at all, so a null one is refused rather than + /// dereferenced. `kind` names the operator the first message speaks of, which is all that sets + /// the entry points' messages apart. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 273804d0d..af144f8a0 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -383,84 +385,106 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - std::string data_table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &data_table_path, - /*table_id=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadSystemTable(identifier, load_identifier, branch)); if (table_path != nullptr) { - *table_path = data_table_path; - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); + *table_path = system_table->GetRootPath(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } -Result> RestCatalog::LoadFormatTable( - const Identifier& identifier) const { - // A system table is already refused by `Catalog::GetFormatTable()`, which every caller goes - // through, so this only has to load. - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - branch = NormalizeBranch(std::move(branch)); - PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - std::string location; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, &location, /*table_id=*/nullptr)); - // A rest catalog holds the schema itself, so everything below the location is data. - return FormatTable::Create(fs_, location, identifier, checked_pointer_cast(schema), - /*location_carries_paimon_metadata=*/false, - /*dynamic_options=*/{}); +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options); } Result> RestCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + return FileStoreTable::Create( + table_file_system, table_path, schema, identifier, uuid, + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2f57e912d..1663b6d2e 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -36,12 +36,15 @@ struct ArrowSchema; namespace paimon { class FileSystem; +class SystemTable; class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to /// `RestApi`; table data is accessed through the file system configured by the /// server-merged options. -class RestCatalog : public Catalog, public VersionManagedCatalog { +class RestCatalog : public Catalog, + public VersionManagedCatalog, + public std::enable_shared_from_this { public: /// Creates the catalog: fetches and merges "/v1/config" from the server configured /// by `CatalogOptions::URI`, then builds the file system from the merged options. @@ -117,12 +120,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// Options merged with the server side config. const std::map& GetOptions() const override; - protected: - /// Loads the location and the schema from one `GetTable` response, so the table cannot be - /// built from a location and a schema that two requests disagreed on. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, @@ -140,6 +137,12 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries, from the same response that carries the data table's schema and path. + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; + /// Converts a `GetTableResponse` into a `TableSchema`, applying the table path, audit /// and branch options. static Result> ToTableSchema( diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 147a908ce..78e1eb529 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,15 +49,24 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -801,7 +810,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); ASSERT_TRUE(databases.empty()); options_[CatalogOptions::METASTORE] = "something-else"; @@ -1061,9 +1070,11 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - ASSERT_EQ("t1$options", table->Name()); + std::shared_ptr system_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, system_table); + ASSERT_EQ("t1$options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1134,8 +1145,9 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); - ASSERT_EQ("main", main_case_branch); + auto main_case_file_store_table = std::dynamic_pointer_cast(main_case_table); + ASSERT_NE(nullptr, main_case_file_store_table); + ASSERT_EQ("main", main_case_file_store_table->CurrentBranch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server @@ -1213,8 +1225,10 @@ TEST_F(RestCatalogTest, CreateRefusesAFormatTableThisClientCouldNotOpen) { // And a schema this library can open is created and opens. ASSERT_OK(create("fine", good, {"dt"}, format_table)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fine"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fine"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); } @@ -1236,7 +1250,7 @@ TEST_F(RestCatalogTest, TableResponseWithoutAPathIsRejected) { // would be checked against an empty one has no boundary at all, so the response is refused // rather than turned into a table. What the response said is not repeated back: a body may // carry credentials, so a failure to read one names the request and nothing else. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "no_path")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "no_path")), "failed to deserialize the response"); ASSERT_NOK_WITH_MSG(catalog->GetTableLocation(Identifier("db1", "no_path")), "failed to deserialize the response"); @@ -1259,8 +1273,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { state_->get_table_requests = 0; } - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fmt"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ("wh1/db1.db/fmt", table->Location()); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); ASSERT_EQ((std::vector{"dt"}), table->PartitionKeys()); @@ -1275,18 +1291,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - - // A managed table stays out of this path, and so does a system table. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1")), "is not a format table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1$snapshots")), - "for system table"); - // The database is checked before the table name is parsed, so a malformed name under "sys" - // is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, + catalog->GetTable(Identifier("db1", "t1"))); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(managed)); } TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { @@ -1312,24 +1320,23 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { // A format table format with no reader here. seed("csv_table", R"({"type": "format-table", "file.format": "csv"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "csv_table")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "csv_table")), "not supported by paimon-cpp yet"); // A row count no file could ever reach. seed("bad_rows", R"({"type": "format-table", "file.format": "parquet", "target-file-row-num": "0"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), - "should be at least 1"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "should be at least 1"); // The failure names the table, which is all a caller holding several has to go on. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); // The structural rules run here too, not only the format-table ones: a partition key that is // not a field of the schema is a table nothing could ever read. seed("bad_partition_key", R"({"type": "format-table", "file.format": "parquet"})", R"(["nosuchfield"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_partition_key")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_partition_key")), "should include all partition fields"); // Two schemas this library cannot open, refused when the table is opened rather than when a @@ -1339,12 +1346,12 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"([{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" // Not a raw string: `TIMESTAMP(6)"` holds the sequence that would end one. " {\"id\": 1, \"name\": \"dt\", \"type\": \"TIMESTAMP(6)\"}]"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "timestamp_partition")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "timestamp_partition")), "cannot be TIMESTAMP/DECIMAL/BLOB"); seed("only_partitions", R"({"type": "format-table", "file.format": "parquet"})", R"(["dt"])", R"([{"id": 0, "name": "dt", "type": "STRING"}])"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "only_partitions")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "only_partitions")), "every one of its columns"); // An option this library does not honour keeps its own status code, so a caller can still @@ -1353,8 +1360,7 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"({"type": "format-table", "file.format": "parquet",)" R"( "metastore.partitioned-table": "true"})", R"(["dt"])", default_fields); - Status not_implemented = - catalog->GetFormatTable(Identifier("db1", "catalog_partitions")).status(); + Status not_implemented = catalog->GetTable(Identifier("db1", "catalog_partitions")).status(); ASSERT_TRUE(not_implemented.IsNotImplemented()) << not_implemented.ToString(); } @@ -1535,6 +1541,59 @@ TEST_F(RestCatalogTest, SystemDatabase) { ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); + std::shared_ptr catalog = std::move(created); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + +TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 1a2f0b7d7..2890330c3 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -35,6 +35,7 @@ #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -115,9 +116,11 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( + return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + /*root_path=*/"", table_schema_, identifier, table_uuid_, + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4cbfda4df..689ca9513 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -216,6 +216,12 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return fields; } + /// The identifier `WriteDanglingBlobViewRows` stamps on row 0: no table exists where the + /// catalog issues it, so the catalog the blob view lookup resolves the reference through + /// reports the identifier not exist, and every read that reaches the reference fails on it. + static constexpr const char* kDanglingUpstreamNotFound = + "Identifier{database='nonexistent_db', object='nonexistent_table'} not exist"; + /// Writes and commits the two rows the blob-view skip tests share: row 0 holds a blob view /// reference to an upstream table that does not exist, row 1 holds no blob view at all. /// Resolving row 0 fails the whole read, so any read the pre-read still reaches it from must @@ -4589,10 +4595,11 @@ TEST_P(BlobTableInteTest, TestBlobViewFailsWhenUpstreamTableAbsent) { ASSERT_OK(WriteDanglingBlobViewRows(table_path)); - // reading fails because the upstream table does not exist at the catalog-issued location + // reading fails because the upstream table does not exist at the catalog-issued location: + // the catalog the lookup resolves the reference through reports its identifier not exist ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); } TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { @@ -4611,7 +4618,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { // without the deletion vector the dangling reference fails the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); ASSERT_OK_AND_ASSIGN(std::string anchor_file_name, DeletionVectorTestHelper::RetrieveAnchorFileName(commit_msgs)); @@ -4655,7 +4662,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceInEveryRowRangeGroup // without deletion vectors both groups' dangling references fail the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // each group's anchor gets a deletion vector dropping its row 0, the dangling one ASSERT_OK_AND_ASSIGN(std::string anchor_file_name0, @@ -4692,7 +4699,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { // reading the whole split fails on the dangling reference of row 0 ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // a predicate keeping only row 1 does not rescue the query: it is evaluated above the split // read, while the pre-read takes none, so row 0's reference is still collected and resolved @@ -4700,7 +4707,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { FieldType::INT, Literal(101)); ASSERT_OK_AND_ASSIGN(auto filtered_plan, ScanTable(table_path, keep_row_1)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), filtered_plan, keep_row_1), - "not found latest schema"); + kDanglingUpstreamNotFound); // dropping the same row through the row ranges does: the pre-read honors them the same way // the main read does, so row 0 is never resolved diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 284cd362f..5aa2f22c3 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -49,6 +49,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -57,6 +58,7 @@ #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" @@ -274,6 +276,10 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); if (predicate) { @@ -282,7 +288,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); @@ -299,7 +305,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -650,17 +656,18 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); + ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -682,17 +689,20 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1002,12 +1012,16 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + table->NewScan(std::move(ro_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); @@ -1018,7 +1032,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1062,11 +1076,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder scan_context_builder(table_path + "$ro"); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1079,7 +1097,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -4173,8 +4191,9 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result> ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, - const std::string& warehouse, const std::map& options) { + const std::string& table_name, const std::shared_ptr& catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -4190,18 +4209,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -4229,8 +4248,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("catalog_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ReadGlobalSystemTable("catalog_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); @@ -4301,7 +4320,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { // Directly test BuildRows { GlobalSystemTableContext ctx; - ctx.catalog = catalog.get(); + ctx.catalog = catalog; ctx.fs = catalog->GetFileSystem(); ctx.warehouse = warehouse; ctx.catalog_options = options; @@ -4310,9 +4329,9 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; } - ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("all_table_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ASSERT_OK_AND_ASSIGN( + auto result, ReadGlobalSystemTable("all_table_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; @@ -4392,7 +4411,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { ArrowSchemaRelease(&no_pk_schema); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + ReadGlobalSystemTable("tables", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1); @@ -4490,8 +4509,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); auto fs = catalog->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, - warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(part_result); ASSERT_TRUE(struct_array); ASSERT_EQ(StructFieldNames(struct_array), @@ -4542,7 +4561,7 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { /*overwrite=*/true)); for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { - ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog, fs, warehouse, options)); } } @@ -4578,7 +4597,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { @@ -4617,7 +4636,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..7c4110257 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -61,6 +61,7 @@ #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/realtime_split.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/data/variant.h" @@ -504,7 +505,7 @@ class RealtimeWriteInteTest : public ::testing::Test { void CreateTable(const std::vector& partition_keys) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys, @@ -516,7 +517,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::vector& primary_keys = {"id"}) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); std::vector table_primary_keys = partition_keys; @@ -2918,10 +2919,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + table_path_ + "$ro", options_)); ScanContextBuilder scan_builder(table_path_ + "$ro"); scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + Result> scan = table->NewScan(std::move(scan_context)); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized"));