refactor(table): split the table hierarchy and materialize the store a table serves

benchmark/benchmark_suite.cpp

@@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name,
                    const std::shared_ptr<arrow::Schema>& schema,
                    const std::vector<std::string>& primary_keys = {}) {
     PAIMON_ASSIGN_OR_RAISE(
-        std::unique_ptr<paimon::Catalog> catalog,
+        std::shared_ptr<paimon::Catalog> catalog,
         AddContext(paimon::Catalog::Create(root_path, options), "create catalog"));
     PAIMON_RETURN_NOT_OK(
         AddContext(catalog->CreateDatabase(db_name, options, true), "create database"));

docs/source/api/format_table.rst

@@ -35,18 +35,16 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`.
 Contexts
 ========
 
-Each of those entry points builds its context from a table path, and the schema under that path
-says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead,
-through the constructor each context builder has for one:
-
-* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
-* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
-* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
-* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
-
-This is the only way to reach a format table whose schema does not live under its own location,
-such as one a REST catalog serves: nothing under the location says that it is a format table, nor
-that what sits below it is data rather than metadata. The table knows both, so a setting that would
-answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them -
-is refused by these builders rather than quietly ignored. Options given at the call still win over
-the ones the table carries, as they do everywhere else.
+Each of those entry points builds its context the way it does for a data table: from the table's
+location, through the builder's usual constructor:
+
+* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)``
+* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)``
+* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string&)``
+* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string&)``
+
+The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or
+``NewCommit()`` - which serves a table whose schema does not live under its own location as well,
+such as one a REST catalog serves: the table carries its schema and the file system it was loaded
+through, so nothing under the location has to answer for them. Options given at the call still win
+over the ones the table carries, as they do everywhere else.

docs/source/user_guide/catalog.rst

@@ -83,7 +83,7 @@ registered on the REST server. The catalog is configured through the
        {"token.provider", "bear"},
        {"token", "<token>"},
    };
-   PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::Catalog> catalog,
+   PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::Catalog> catalog,
                           paimon::Catalog::Create(/*root_path=*/"my_instance", options));
 
 For DLF, configure one credential source. Static AK/SK credentials, an optional
@@ -122,22 +122,19 @@ without an atomic snapshot-file rename:
 
 .. code-block:: cpp
 
-   // Writers and committers share ownership of the catalog.
-   std::shared_ptr<paimon::Catalog> shared_catalog(std::move(catalog));
-
    // Load the table's identity before preparing changes.
    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::Table> table,
-                          shared_catalog->GetTable(paimon::Identifier("db", "tbl")));
+                          catalog->GetTable(paimon::Identifier("db", "tbl")));
 
    paimon::CommitContextBuilder builder(table_path, "commit-user");
-   builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl"));
+   builder.WithCatalog(catalog, paimon::Identifier("db", "tbl"));
    // CatalogUuid() stays null when the server supplies no id.
    if (std::optional<std::string> table_uuid = table->CatalogUuid()) {
        builder.WithTableId(table_uuid.value());
    }
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::CommitContext> context, builder.Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::FileStoreCommit> commit,
-                          paimon::FileStoreCommit::Create(std::move(context)));
+                          table->NewCommit(std::move(context)));
    PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages));
 
 Configure native writers with the same catalog and identifier. They load the
@@ -147,7 +144,7 @@ offsets when a real-time writer is created:
 .. code-block:: cpp
 
    paimon::WriteContextBuilder write_builder(table_path, "commit-user");
-   write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl"));
+   write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl"));
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::WriteContext> write_context,
                           write_builder.Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::FileStoreWrite> writer,
@@ -166,8 +163,8 @@ Both builders use the file system of the table being written or committed for
 manifests and data, so a catalog that issues temporary credentials per table
 serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides
 it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override
-file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)``
-instead.
+file-system selection. A format table is written through the table's own
+``NewWrite`` instead; see :doc:`format_table`.
 
 A file system passed to ``WithFileSystem`` is used exactly as it is: neither the
 built-in file systems nor the table's own credentials are involved, so a custom
@@ -183,7 +180,7 @@ returns credentials that are still valid and reloads them before they expire.
                                                                  "oss://bucket/tbl", options));
    // MyFileSystem merges provider->GetCredentials() over its own options per access.
    auto my_fs = std::make_shared<MyFileSystem>(provider);
-   write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl"))
+   write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl"))
        .WithFileSystem(my_fs);
 
 The snapshot being committed carries a uuid generated on the client, and the
@@ -262,18 +259,15 @@ for that table's file system and passing it in through ``WithFileSystem``:
 
 .. code-block:: cpp
 
-   // Readers and scanners share ownership of the catalog.
-   std::shared_ptr<paimon::Catalog> shared_catalog(std::move(catalog));
-
    paimon::ScanContextBuilder scan_builder(table_path);
-   scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl"));
+   scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl"));
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ScanContext> scan_context,
                           scan_builder.Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableScan> scan,
                           paimon::TableScan::Create(std::move(scan_context)));
 
    paimon::ReadContextBuilder read_builder(table_path);
-   read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl"));
+   read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl"));
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ReadContext> read_context,
                           read_builder.Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableRead> read,
@@ -285,10 +279,9 @@ snapshots, manifests and data under the table path, so it signs every access wit
 the table's credentials and reloads them as they expire. An explicit
 ``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that
 issues no per-table credentials returns its catalog-level file system, so this is
-also how you read with the catalog's own object-store credentials. For a format
-table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)``
-instead; ``WithCatalog`` is rejected because the table already carries the file
-system it was loaded through.
+also how you read with the catalog's own object-store credentials. A format table
+is read through the table's own ``NewRead`` or ``NewScan`` instead; see
+:doc:`format_table`.
 
 The C++ REST catalog covers the database, table, snapshot and commit operations
 of the ``Catalog`` API. The parts of the Java REST catalog that have no C++

docs/source/user_guide/format_table.rst

@@ -40,37 +40,43 @@ format table is an insert.
 
 Reading and writing
 -------------------
-A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed
-table; use :cpp:func:`Catalog::GetFormatTable` instead.
-
-Reading and writing go through the entry points every other table uses: ``TableScan::Create``,
-``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its
-usual context builder. Each reads the table's schema - from under the table path, or from the one
-the context carries - and dispatches to a format table when its ``type`` says so.
-
-That path needs a schema under the table's own location, which is what every table a file system
-catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the
-REST catalog, has none: nothing under the location says that it is a format table, nor that
-``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written
-by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has
-a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and
+A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table:
+its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes,
+through the ``Table`` interface the two kinds of table share.
+
+Reading and writing go through the table itself: a caller holding one calls ``NewScan()``,
+``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds from
+the table's location. This is what Java serves through ``FormatTable.newReadBuilder()`` and
 ``newBatchWriteBuilder()``:
 
 .. code-block:: cpp
 
-   PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::FormatTable> table,
-                          catalog->GetFormatTable(paimon::Identifier("db", "tbl")));
-   paimon::ScanContextBuilder scan_builder(table);
+   PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::Table> loaded,
+                          catalog->GetTable(paimon::Identifier("db", "tbl")));
+   // The table's `type` option is `format-table`, so this is a `FormatTable`.
+   std::shared_ptr<paimon::FormatTable> table =
+       std::dynamic_pointer_cast<paimon::FormatTable>(loaded);
+   paimon::ScanContextBuilder scan_builder(table->Location());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ScanContext> scan_context,
                           scan_builder.Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableScan> scan,
-                          paimon::TableScan::Create(std::move(scan_context)));
+                          table->NewScan(std::move(scan_context)));
 
-The table already carries its schema and the file system it was loaded through, so a context built
-from one refuses ``SetTableSchema()``, ``WithFileSystem()``,
-``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options
+Because the operator comes from the table itself, this serves a table whose schema does not live
+under its own location as well, such as one a REST catalog serves: the table carries its schema and
+the file system it was loaded through, so nothing under the location has to answer for them. Options
 given at the call still win over the ones the schema stored, as they do everywhere else.
 
+The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create``
+and ``FileStoreCommit::Create`` - serve a data table only: a load from a path is the data table's
+way in, and a format table the schema under the path names is refused with a message pointing at
+the catalog and the table's own operators above.
+
+``FormatTable`` is a derived class of ``Table``, so a caller holding one through a
+``std::shared_ptr<paimon::Table>`` calls the same operators on it without the cast: they come back
+for the table it is, whatever kind it turns out to be. This is the C++ side of what Java serves
+through one ``Table.newReadBuilder()`` for both kinds of table.
+
 A batch comes back in the table's column order, or the projection's when the read names one,
 behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for
 and in what order is its own business: they are matched to the table's by name, as Java matches
@@ -89,12 +95,10 @@ outside the projection is never read, which is how Java scopes the same checks.
 named here only to describe what happens. A caller never needs them: a plan comes back as
 ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``.
 
-A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the
-protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say
-its metadata lives under the table location, and the REST catalog to take the location and the
-schema from one response instead of two that could disagree. A catalog that does not override it
-still serves format tables, by reading the location and the schema through the virtuals every
-catalog has.
+Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the
+schema under the table location, so it says the location carries the table's own metadata and a
+scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the
+location and the schema from one response, so the two cannot disagree.
 
 ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by
 listing directories; every other table type returns ``NotImplemented`` for now.

examples/read_write_demo.cpp

@@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name,
                                                   {paimon::Options::FILE_SYSTEM, "local"}};
 
     // create table
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::Catalog> catalog,
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::Catalog> catalog,
                            paimon::Catalog::Create(root_path, options));
     PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false));
     arrow::FieldVector fields = {

include/paimon/catalog/catalog.h

@@ -40,7 +40,6 @@ namespace paimon {
 using Instant = std::variant<std::string, int64_t>;
 
 class Database;
-class FormatTable;
 class Table;
 class View;
 class Schema;
@@ -69,8 +68,11 @@ class PAIMON_EXPORT Catalog {
     ///                    schemes, including the file systems it builds from per-table data
     ///                    tokens, keeps routing each scheme to its backend. Ignored when
     ///                    `file_system` is supplied, which serves every scheme itself.
-    /// @return A result containing a unique pointer to a `Catalog` instance, or an error status.
-    static Result<std::unique_ptr<Catalog>> Create(
+    /// @return A result containing a shared pointer to a `Catalog` instance, or an error status.
+    /// @note Shared rather than unique ownership: a global system table of the `sys` database
+    ///       builds its rows lazily through the catalog it was handed, so the table keeps the
+    ///       catalog alive. A catalog built by this factory always satisfies that.
+    static Result<std::shared_ptr<Catalog>> Create(
         const std::string& root_path, const std::map<std::string, std::string>& options,
         const std::shared_ptr<FileSystem>& file_system = nullptr,
         const std::map<std::string, std::string>& fs_scheme_to_identifier_map = {});
@@ -148,6 +150,17 @@ class PAIMON_EXPORT Catalog {
 
     /// Gets a table.
     ///
+    /// The table's `type` option decides what is handed back: a `format-table` comes back as the
+    /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and
+    /// every other implemented type as the managed table it is, while a `T$system` identifier names
+    /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and
+    /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones.
+    ///
+    /// A global system table of the "sys" database reads its rows lazily through the catalog it
+    /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands
+    /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as
+    /// through a `std::unique_ptr`, cannot be kept and is refused by name.
+    ///
     /// @param identifier Identifier of the table to get.
     /// @return A result containing the table, or an error status.
     virtual Result<std::shared_ptr<Table>> GetTable(const Identifier& identifier) const = 0;
@@ -246,33 +259,6 @@ class PAIMON_EXPORT Catalog {
     /// @note Implementations returning true must also implement the internal
     /// `VersionManagedCatalog`.
     virtual bool SupportsVersionManagement() const;
-
-    /// Gets a format table: a directory of data files laid out like a standard Hive table.
-    ///
-    /// A format table carries no snapshots and no manifests, so it is loaded through its own
-    /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`,
-    /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses:
-    /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or
-    /// `CommitContextBuilder`, each of which takes one.
-    ///
-    /// @param identifier Identifier of the table to get.
-    /// @return A result containing the format table, or an error status if the table does not
-    /// exist or its `type` option is not `format-table`.
-    Result<std::shared_ptr<FormatTable>> GetFormatTable(const Identifier& identifier) const;
-
- protected:
-    /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back.
-    ///
-    /// The default reads the location and the schema through the virtuals above - two requests
-    /// that can disagree - and treats every directory below the location as table content. An
-    /// override answers both questions directly: whether location and schema come back in one
-    /// round trip, and whether this catalog keeps the table's metadata under the table path.
-    ///
-    /// @param identifier Identifier of the table to load.
-    /// @return A result containing the format table, or an error status if the table does not
-    /// exist or its `type` option is not `format-table`.
-    virtual Result<std::shared_ptr<FormatTable>> LoadFormatTable(
-        const Identifier& identifier) const;
 };
 
 }  // namespace paimon