From 360ac6f8b71aa135bc761d9902c9dec66f051e2d Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 18:08:42 +0800 Subject: [PATCH 01/35] [feat](fluss) Add the fluss connector module and its remote-call seam Introduces fe-connector-fluss: the plugin identity (provider, services entry, plugin zip), the catalog property contract, and FlussAdminOps -- the single interface through which this connector talks to a fluss cluster. FlussAdminOps exists so that metadata mapping and, later, split generation stay pure functions of what the cluster reports. That is what lets a unit test drive the states that decide correctness (an empty bucket, a lake offset that caught up with the log, a partition that only the lake still has) without a mocking framework, which the fe-connector modules do not carry. Notes on the fluss client that shape the pom: - it ships as one unrelocated fat jar carrying fluss-common/-rpc, frocksdbjni, commons-lang3 and zstd, so fluss-common must NOT be declared separately and frocksdbjni cannot be excluded; - it declares Arrow `provided`, so a consumer reaching fluss's ARROW log format has to add Arrow itself. Metadata and planning do not, so this module does not. Fluss errors travel unwrapped: scan planning has to discriminate on LakeTableSnapshotNotExistException to decide whether a union read is possible, and wrapping would turn that into string matching. The TCCL pin sits at connection creation because that is where the client's own threads are born and inherit a context classloader; pinning once there covers them for life. Version is pinned by one property in fe/pom.xml for every fluss consumer: the FE plans splits against the same client the BE scanner will read with, and the scanner links fluss classes marked @Internal, so a skew breaks the read path at runtime rather than at build time. Co-Authored-By: Claude Opus 5 (1M context) --- fe/fe-connector/fe-connector-fluss/pom.xml | 108 +++++++++++ .../src/main/assembly/plugin-zip.xml | 65 +++++++ .../fluss/ConnectionBackedFlussAdminOps.java | 169 ++++++++++++++++++ .../doris/connector/fluss/FlussAdminOps.java | 97 ++++++++++ .../doris/connector/fluss/FlussConnector.java | 125 +++++++++++++ .../fluss/FlussConnectorMetadata.java | 53 ++++++ .../fluss/FlussConnectorProperties.java | 160 +++++++++++++++++ .../fluss/FlussConnectorProvider.java | 46 +++++ ...ache.doris.connector.spi.ConnectorProvider | 1 + .../fluss/FlussConnectorMetadataTest.java | 53 ++++++ .../fluss/FlussConnectorPropertiesTest.java | 118 ++++++++++++ .../fluss/FlussConnectorProviderTest.java | 93 ++++++++++ .../fluss/RecordingFlussAdminOps.java | 133 ++++++++++++++ fe/fe-connector/pom.xml | 1 + fe/pom.xml | 11 ++ 15 files changed, 1233 insertions(+) create mode 100644 fe/fe-connector/fe-connector-fluss/pom.xml create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml new file mode 100644 index 00000000000000..0633d1e1b0fdc4 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-fluss + jar + Doris FE Connector - Fluss + + Fluss connector plugin for Doris FE. + Provides metadata operations (list databases, list tables, get schema) and read-side scan + planning for Fluss clusters via the connector SPI. + + + + + ${project.groupId} + fe-connector-spi + ${project.version} + + + + + ${project.groupId} + fe-connector-api + ${project.version} + + + + + org.apache.fluss + fluss-client + ${fluss.version} + + + + org.junit.jupiter + junit-jupiter + test + + + + + doris-fe-connector-fluss + + + maven-assembly-plugin + + false + + src/main/assembly/plugin-zip.xml + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml b/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..f10c88f571a01a --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml @@ -0,0 +1,65 @@ + + + + + plugin + + zip + + false + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + + + + /lib + false + runtime + + org.apache.doris:fe-connector-api + org.apache.doris:fe-connector-spi + org.apache.doris:fe-extension-spi + org.apache.doris:fe-filesystem-api + org.apache.doris:fe-thrift + org.apache.thrift:libthrift + + org.apache.logging.log4j:* + org.slf4j:* + + + + diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java new file mode 100644 index 00000000000000..4dc6d7b80cf96a --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * {@link FlussAdminOps} over a live fluss {@link Connection}: the one place in this connector that + * deals with futures, timeouts and error wrapping. + * + *

The connection is owned by {@link FlussConnector} and outlives this object, which is created per + * statement. {@code Connection#getAdmin()} is a memoized, connection-scoped singleton, so it is fetched + * per call and never closed here — closing it would pull the admin out from under every other statement + * on the same catalog. + */ +public class ConnectionBackedFlussAdminOps implements FlussAdminOps { + + /** + * How long any single metadata call may take before it is abandoned. These are small RPCs against + * the coordinator, so a wait this long means the cluster is unreachable rather than busy, and the + * caller is a query that should say so instead of hanging. + */ + private static final long TIMEOUT_MS = 60_000L; + + private final Connection connection; + private final String catalogName; + private final String bootstrapServers; + + public ConnectionBackedFlussAdminOps(Connection connection, String catalogName, String bootstrapServers) { + this.connection = connection; + this.catalogName = catalogName; + this.bootstrapServers = bootstrapServers; + } + + @Override + public List listDatabases() { + return await(admin().listDatabases(), "listDatabases"); + } + + @Override + public boolean databaseExists(String databaseName) { + return await(admin().databaseExists(databaseName), "databaseExists(" + databaseName + ")"); + } + + @Override + public List listTables(String databaseName) { + return await(admin().listTables(databaseName), "listTables(" + databaseName + ")"); + } + + @Override + public boolean tableExists(TablePath tablePath) { + return await(admin().tableExists(tablePath), "tableExists(" + tablePath + ")"); + } + + @Override + public TableInfo getTableInfo(TablePath tablePath) { + return await(admin().getTableInfo(tablePath), "getTableInfo(" + tablePath + ")"); + } + + @Override + public List listPartitionInfos(TablePath tablePath) { + return await(admin().listPartitionInfos(tablePath), "listPartitionInfos(" + tablePath + ")"); + } + + @Override + public List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec) { + return await(admin().listPartitionInfos(tablePath, partialPartitionSpec), + "listPartitionInfos(" + tablePath + ", " + partialPartitionSpec + ")"); + } + + @Override + public TableStats getTableStats(TablePath tablePath) { + return await(admin().getTableStats(tablePath), "getTableStats(" + tablePath + ")"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath) { + return await(admin().getLatestKvSnapshots(tablePath), "getLatestKvSnapshots(" + tablePath + ")"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName) { + return await(admin().getLatestKvSnapshots(tablePath, partitionName), + "getLatestKvSnapshots(" + tablePath + ", " + partitionName + ")"); + } + + @Override + public LakeSnapshot getReadableLakeSnapshot(TablePath tablePath) { + return await(admin().getReadableLakeSnapshot(tablePath), "getReadableLakeSnapshot(" + tablePath + ")"); + } + + @Override + public Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec) { + return await(admin().listOffsets(tablePath, buckets, offsetSpec).all(), + "listOffsets(" + tablePath + ", " + buckets + ")"); + } + + @Override + public Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + return await(admin().listOffsets(tablePath, partitionName, buckets, offsetSpec).all(), + "listOffsets(" + tablePath + ", " + partitionName + ", " + buckets + ")"); + } + + private Admin admin() { + return connection.getAdmin(); + } + + private T await(CompletableFuture future, String operation) { + try { + return future.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DorisConnectorException(describe(operation, "was interrupted"), e); + } catch (TimeoutException e) { + future.cancel(true); + throw new DorisConnectorException(describe(operation, "timed out after " + TIMEOUT_MS + " ms"), e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + // Fluss's own errors (ApiException subclasses) are RuntimeExceptions, and callers + // discriminate on their type — LakeTableSnapshotNotExistException is what tells scan + // planning that this table has nothing readable in the lake yet. Wrapping them here + // would turn that decision into string matching, so they travel unchanged. + throw (RuntimeException) cause; + } + throw new DorisConnectorException(describe(operation, "failed"), cause == null ? e : cause); + } + } + + private String describe(String operation, String outcome) { + return "fluss catalog '" + catalogName + "': " + operation + " " + outcome + + " (" + FlussConnectorProperties.BOOTSTRAP_SERVERS + "=" + bootstrapServers + ")"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java new file mode 100644 index 00000000000000..b544331ea04f39 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Every remote call this connector makes to a fluss cluster, as one synchronous interface. + * + *

It exists to make the connector testable without a mocking framework. Metadata mapping and, above + * all, split generation are pure functions of what these methods return — offsets, snapshot ids, + * partition lists — so a recording implementation lets a unit test drive the interesting boundaries + * (an empty bucket, a lake offset that has caught up with the log, a partition that exists only in the + * lake) directly, instead of trying to coax a live cluster into each of those states. The + * connection-backed implementation is then the only place that has to worry about futures, timeouts + * and error wrapping, and integration tests against a real cluster cover it. + * + *

Signatures mirror {@code org.apache.fluss.client.admin.Admin} one for one, minus the asynchrony: + * every method blocks with a timeout and returns the value, so nothing downstream handles a + * {@link java.util.concurrent.CompletableFuture}. Fluss's own exceptions (subclasses of + * {@code org.apache.fluss.exception.ApiException}) propagate unwrapped — callers discriminate on them, + * notably {@code LakeTableSnapshotNotExistException} when deciding whether a union read is possible. + * + *

Implementations are NOT thread-safe and are scoped to one statement. + */ +public interface FlussAdminOps { + + List listDatabases(); + + boolean databaseExists(String databaseName); + + List listTables(String databaseName); + + boolean tableExists(TablePath tablePath); + + TableInfo getTableInfo(TablePath tablePath); + + List listPartitionInfos(TablePath tablePath); + + /** Server-side partial partition pruning: only partitions matching {@code partialPartitionSpec}. */ + List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec); + + TableStats getTableStats(TablePath tablePath); + + /** Latest kv snapshot per bucket of a primary-key table, with the log offset each snapshot ends at. */ + KvSnapshots getLatestKvSnapshots(TablePath tablePath); + + /** As above, for one partition of a partitioned primary-key table. */ + KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName); + + /** + * The lake snapshot a reader may actually read, together with the per-bucket log offset the lake + * ends at. + * + *

Deliberately not the latest lake snapshot: data that tiering has just committed can + * still be unreadable in the lake (a paimon deletion-vector table keeps it in L0 for a while), and + * a reader that took the latest snapshot id would skip the log covering it and lose rows. Both + * halves — snapshot id and offsets — come from this one call so they cannot disagree. + * + * @throws org.apache.fluss.exception.LakeTableSnapshotNotExistException if nothing is readable in + * the lake yet, which is the normal state of a table whose tiering has not committed once + */ + LakeSnapshot getReadableLakeSnapshot(TablePath tablePath); + + /** Offsets for {@code buckets} of a non-partitioned table, keyed by bucket id. */ + Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec); + + /** Offsets for {@code buckets} of one partition, keyed by bucket id. */ + Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec); +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java new file mode 100644 index 00000000000000..5d79a533472a9b --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTestResult; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.config.Configuration; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Fluss connector: one instance per catalog, owning that catalog's single fluss {@link Connection}. + * + *

The connection is the expensive, thread-safe, long-lived object (it carries the RPC client, + * metadata updater and their threads); {@code Admin} handles taken from it are cheap and memoized by + * the connection itself. So the connection is created once, lazily, and closed when the catalog goes + * away, while each statement gets its own {@link ConnectorMetadata} over a fresh + * {@link ConnectionBackedFlussAdminOps} view of it. + */ +public class FlussConnector implements Connector { + + private final Map properties; + private final String catalogName; + + private volatile Connection connection; + + public FlussConnector(Map properties, ConnectorContext context) { + FlussConnectorProperties.validate(properties); + this.properties = Collections.unmodifiableMap(new HashMap<>(properties)); + this.catalogName = context.getCatalogName(); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return new FlussConnectorMetadata(adminOps()); + } + + @Override + public ConnectorTestResult testConnection(ConnectorSession session) { + try { + adminOps().listDatabases(); + return ConnectorTestResult.success(); + } catch (Exception e) { + return ConnectorTestResult.failure("Fluss connectivity test failed: " + e.getMessage()); + } + } + + @Override + public void close() throws IOException { + Connection toClose = connection; + connection = null; + if (toClose == null) { + return; + } + try { + toClose.close(); + } catch (Exception e) { + throw new IOException("Failed to close the fluss connection of catalog '" + catalogName + "'", e); + } + } + + private FlussAdminOps adminOps() { + return new ConnectionBackedFlussAdminOps(getOrCreateConnection(), catalogName, + FlussConnectorProperties.bootstrapServers(properties)); + } + + private Connection getOrCreateConnection() { + if (connection == null) { + synchronized (this) { + if (connection == null) { + connection = createConnection(); + } + } + } + return connection; + } + + private Connection createConnection() { + Configuration config = new Configuration(); + FlussConnectorProperties.toFlussClientConfig(properties).forEach(config::setString); + + // TCCL pin, and this is the locus that matters: creating the connection is what spawns the + // fluss client's own threads (netty IO, metadata updater), and a thread inherits the context + // classloader of whoever created it. Started under the engine's loader they would resolve + // fluss classes against fe-core instead of this child-first plugin, where fluss does not exist + // at all. Pinning here fixes every thread the connection owns for its whole life, which is why + // the per-call admin path needs no pin of its own. + ClassLoader callerLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + try { + return ConnectionFactory.createConnection(config); + } catch (RuntimeException e) { + throw new DorisConnectorException("Failed to connect fluss catalog '" + catalogName + "' (" + + FlussConnectorProperties.BOOTSTRAP_SERVERS + "=" + + FlussConnectorProperties.bootstrapServers(properties) + ")", e); + } finally { + Thread.currentThread().setContextClassLoader(callerLoader); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java new file mode 100644 index 00000000000000..6a050683b7a401 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; + +import java.util.List; + +/** + * Fluss metadata for one statement: a thin mapping from the connector SPI onto {@link FlussAdminOps}. + * + *

Fluss has a real two-level namespace (database, table), so the listing calls are direct + * pass-throughs and carry no Doris-side naming convention. + */ +public class FlussConnectorMetadata implements ConnectorMetadata { + + private final FlussAdminOps adminOps; + + public FlussConnectorMetadata(FlussAdminOps adminOps) { + this.adminOps = adminOps; + } + + @Override + public List listDatabaseNames(ConnectorSession session) { + return adminOps.listDatabases(); + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return adminOps.databaseExists(dbName); + } + + @Override + public List listTableNames(ConnectorSession session, String dbName) { + return adminOps.listTables(dbName); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java new file mode 100644 index 00000000000000..1f391465fab25a --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; + +/** + * The catalog properties a fluss catalog accepts, and their translation into a fluss client + * configuration. + * + *

Naming rule. Everything a user may write is prefixed {@code fluss.}, and every such key + * except the Doris-only ones listed in {@link #isDorisOnly(String)} is handed to the fluss client + * verbatim with that prefix stripped. So {@code fluss.bootstrap.servers} becomes fluss's + * {@code bootstrap.servers} and {@code fluss.client.security.protocol} becomes + * {@code client.security.protocol}. Nothing here enumerates the fluss client's own option set: fluss + * owns those names and adds to them between releases, and a Doris-side allowlist would silently + * reject options that fluss understands perfectly well. + * + *

No lake/paimon connection property belongs here. The tiering service writes the lake catalog + * configuration into each datalake-enabled table's own properties ({@code table.datalake.paimon.*}), + * so union read reads it from the table, never from the catalog. + */ +public final class FlussConnectorProperties { + + /** Prefix every fluss catalog property carries; also what is stripped on the way to the client. */ + public static final String PROPERTY_PREFIX = "fluss."; + + /** Required. Comma-separated {@code host:port} list used to bootstrap the fluss cluster. */ + public static final String BOOTSTRAP_SERVERS = "fluss.bootstrap.servers"; + + /** + * Optional. How a scan of a datalake-enabled table combines the paimon lake with the fluss log. + * + *

Doris-only: it selects a planning strategy and is never passed to the fluss client. The + * three values exist because "did this query actually read the lake?" is otherwise unobservable — + * {@code auto} silently falls back to a fluss-only read when the lake has no readable snapshot + * yet, which makes a union-read regression test pass for the wrong reason. {@code required} turns + * that fallback into an error and {@code disabled} forces the fluss-only path, so a test can pin + * both sides of the comparison. + */ + public static final String UNION_READ_MODE = "fluss.union_read.mode"; + + /** Value set of {@link #UNION_READ_MODE}. */ + public enum UnionReadMode { + /** Union read when the lake has a readable snapshot, fluss-only when it does not. */ + AUTO, + /** Union read, or fail: a datalake table with no readable lake snapshot is an error. */ + REQUIRED, + /** Never union read: scan fluss only, even for a datalake table with a lake snapshot. */ + DISABLED; + + static UnionReadMode parse(String raw) { + for (UnionReadMode mode : values()) { + if (mode.name().equalsIgnoreCase(raw.trim())) { + return mode; + } + } + throw new IllegalArgumentException("Invalid value '" + raw + "' for property '" + + UNION_READ_MODE + "'; expected one of auto, required, disabled"); + } + + /** The lower-case spelling users write, and what {@code appendExplainInfo} prints. */ + public String propertyValue() { + return name().toLowerCase(Locale.ROOT); + } + } + + private FlussConnectorProperties() { + } + + /** + * Fails on a catalog that cannot work, at {@code CREATE CATALOG} time rather than at first query. + * Only checks what is decidable without touching the cluster; reachability is + * {@code testConnection}'s job. + */ + public static void validate(Map properties) { + validateBootstrapServers(bootstrapServers(properties)); + unionReadMode(properties); + } + + /** The declared bootstrap servers, or the empty string when the property is absent. */ + public static String bootstrapServers(Map properties) { + String value = properties.get(BOOTSTRAP_SERVERS); + return value == null ? "" : value.trim(); + } + + /** The declared union-read mode, {@link UnionReadMode#AUTO} when the property is absent. */ + public static UnionReadMode unionReadMode(Map properties) { + String value = properties.get(UNION_READ_MODE); + return value == null ? UnionReadMode.AUTO : UnionReadMode.parse(value); + } + + /** + * The fluss client configuration this catalog describes: every {@code fluss.}-prefixed property + * that is not Doris-only, with the prefix stripped. + * + *

Returned sorted so that a configuration is printable and comparable in a test without the map + * iteration order leaking in. + */ + public static Map toFlussClientConfig(Map properties) { + Map config = new TreeMap<>(); + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey(); + if (!key.startsWith(PROPERTY_PREFIX) || isDorisOnly(key)) { + continue; + } + config.put(key.substring(PROPERTY_PREFIX.length()), entry.getValue()); + } + return config; + } + + /** Whether {@code key} configures Doris's own behaviour and must not reach the fluss client. */ + private static boolean isDorisOnly(String key) { + return UNION_READ_MODE.equals(key); + } + + private static void validateBootstrapServers(String value) { + if (value.isEmpty()) { + throw new IllegalArgumentException( + "Required property '" + BOOTSTRAP_SERVERS + "' is missing"); + } + for (String entry : value.split(",", -1)) { + String server = entry.trim(); + // lastIndexOf, not indexOf: a bracketed IPv6 literal ("[::1]:9123") contains colons of its own. + int colon = server.lastIndexOf(':'); + if (colon <= 0 || colon == server.length() - 1) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a comma-separated host:port list"); + } + int port; + try { + port = Integer.parseInt(server.substring(colon + 1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid port in '" + server + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a number between 1 and 65535"); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid port in '" + server + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a number between 1 and 65535"); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java new file mode 100644 index 00000000000000..8b5233be836ad8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import java.util.Map; + +/** + * SPI entry point for the fluss connector. + * Discovered via META-INF/services/org.apache.doris.connector.spi.ConnectorProvider. + */ +public class FlussConnectorProvider implements ConnectorProvider { + + @Override + public String getType() { + return "fluss"; + } + + @Override + public void validateProperties(Map properties) { + FlussConnectorProperties.validate(properties); + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + return new FlussConnector(properties, context); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider b/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider new file mode 100644 index 00000000000000..5a743650899591 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider @@ -0,0 +1 @@ +org.apache.doris.connector.fluss.FlussConnectorProvider diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java new file mode 100644 index 00000000000000..dbaee1b154fe6f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +/** + * Covers the namespace listing surface of the fluss metadata. + */ +public class FlussConnectorMetadataTest { + + @Test + public void listingIsPassedThroughToTheClusterUnchanged() { + // Fluss already has database/table names in Doris's own shape, so nothing here may invent, + // filter or re-case a name: what the cluster reports is what SHOW DATABASES / SHOW TABLES show. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.databases = Arrays.asList("fluss_db", "MixedCase"); + adminOps.tablesByDatabase.put("fluss_db", Arrays.asList("log_table", "pk_table")); + adminOps.tablesByDatabase.put("empty_db", Collections.emptyList()); + + FlussConnectorMetadata metadata = new FlussConnectorMetadata(adminOps); + + Assertions.assertEquals(Arrays.asList("fluss_db", "MixedCase"), metadata.listDatabaseNames(null)); + Assertions.assertEquals(Arrays.asList("log_table", "pk_table"), metadata.listTableNames(null, "fluss_db")); + Assertions.assertEquals(Collections.emptyList(), metadata.listTableNames(null, "empty_db")); + Assertions.assertTrue(metadata.databaseExists(null, "fluss_db")); + Assertions.assertFalse(metadata.databaseExists(null, "absent_db")); + + Assertions.assertEquals( + Arrays.asList("listDatabases()", "listTables(fluss_db)", "listTables(empty_db)", + "databaseExists(fluss_db)", "databaseExists(absent_db)"), + adminOps.calls); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java new file mode 100644 index 00000000000000..b70c862da47531 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.fluss.FlussConnectorProperties.UnionReadMode; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Pins the catalog-property contract of a fluss catalog: what CREATE CATALOG must reject, and exactly + * which properties reach the fluss client. + */ +public class FlussConnectorPropertiesTest { + + private static Map props(String... keyValues) { + Map map = new HashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + map.put(keyValues[i], keyValues[i + 1]); + } + return map; + } + + @Test + public void bootstrapServersIsRequired() { + // A catalog with no bootstrap servers can never answer a query, so it must fail at CREATE + // CATALOG rather than at the user's first SELECT. + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(props())); + Assertions.assertTrue(e.getMessage().contains(FlussConnectorProperties.BOOTSTRAP_SERVERS), + "message should name the missing property, was: " + e.getMessage()); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(props(FlussConnectorProperties.BOOTSTRAP_SERVERS, " "))); + } + + @Test + public void bootstrapServersMustBeHostPortPairs() { + // Each rejected form is one a user actually writes: a bare host, a non-numeric port, a port out + // of range, and a trailing empty element from a stray comma. + for (String bad : new String[] {"localhost", "localhost:", "localhost:abc", "localhost:0", + "localhost:65536", "host1:9123,"}) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate( + props(FlussConnectorProperties.BOOTSTRAP_SERVERS, bad)), + "expected '" + bad + "' to be rejected"); + } + } + + @Test + public void bootstrapServersAcceptsListsAndIpv6() { + // The IPv6 case is why the port is split at the LAST colon, not the first. + for (String good : new String[] {"localhost:9123", " host1:9123 , host2:9124 ", "[::1]:9123"}) { + FlussConnectorProperties.validate(props(FlussConnectorProperties.BOOTSTRAP_SERVERS, good)); + } + } + + @Test + public void unionReadModeDefaultsToAutoAndIsCaseInsensitive() { + Assertions.assertEquals(UnionReadMode.AUTO, FlussConnectorProperties.unionReadMode(props())); + Assertions.assertEquals(UnionReadMode.REQUIRED, + FlussConnectorProperties.unionReadMode(props(FlussConnectorProperties.UNION_READ_MODE, "ReQuIrEd"))); + Assertions.assertEquals(UnionReadMode.DISABLED, + FlussConnectorProperties.unionReadMode(props(FlussConnectorProperties.UNION_READ_MODE, " disabled "))); + } + + @Test + public void unionReadModeRejectsUnknownValueAtCreateCatalog() { + // A typo here would otherwise degrade silently to whatever the default is, and the difference + // between auto and required is only visible as "the query returned fewer rows than it should". + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + FlussConnectorProperties.UNION_READ_MODE, "enabled"); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(properties)); + Assertions.assertTrue(e.getMessage().contains("auto, required, disabled"), + "message should list the accepted values, was: " + e.getMessage()); + } + + @Test + public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() { + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + "fluss.client.security.protocol", "sasl", + FlussConnectorProperties.UNION_READ_MODE, "required", + "type", "fluss", + "warehouse", "s3://ignored"); + + Map config = FlussConnectorProperties.toFlussClientConfig(properties); + + // bootstrap.servers and client.* arrive under fluss's own names; the Doris-only union-read + // switch and every non-fluss catalog property stay behind. The engine's own keys ("type") and + // other connectors' keys ("warehouse") are not fluss options and must not be handed over as if + // they were — the fluss config is not a place to dump whatever the catalog happened to carry. + Map expected = new HashMap<>(); + expected.put("bootstrap.servers", "localhost:9123"); + expected.put("client.security.protocol", "sasl"); + Assertions.assertEquals(expected, config); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java new file mode 100644 index 00000000000000..8223f1e2544e80 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +/** + * Guards the fluss connector's identity and its discovery wiring. + */ +public class FlussConnectorProviderTest { + + private static ConnectorContext context() { + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }; + } + + @Test + public void typeIsFluss() { + // The type name is what CREATE CATALOG routes on and what every statement-scope namespace is + // prefixed with, so it is API: renaming it breaks existing catalogs. + Assertions.assertEquals("fluss", new FlussConnectorProvider().getType()); + Assertions.assertTrue(new FlussConnectorProvider().isStandaloneCatalogType()); + } + + @Test + public void providerIsDiscoverableThroughServiceLoader() { + // The META-INF/services line is what makes the plugin loadable at all; a typo in it is invisible + // until FE starts and the catalog type simply does not exist. + boolean found = false; + for (ConnectorProvider provider : ServiceLoader.load(ConnectorProvider.class, + FlussConnectorProviderTest.class.getClassLoader())) { + if (provider instanceof FlussConnectorProvider) { + found = true; + } + } + Assertions.assertTrue(found, "FlussConnectorProvider must be registered in META-INF/services"); + } + + @Test + public void createValidatesPropertiesBeforeHandingBackACatalog() { + // Validation must also happen on the create path, not only in validateProperties: a catalog + // restored from the FE image at startup goes straight through create. + Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussConnectorProvider().create(new HashMap<>(), context())); + } + + @Test + public void createBuildsAFlussConnectorWithoutTouchingTheCluster() throws IOException { + // Constructing a catalog must stay offline — FE creates connectors while replaying its image, + // where an unreachable cluster must not block startup. The fluss connection is therefore lazy, + // and closing a connector that never connected must be a no-op rather than an NPE. + Map properties = new HashMap<>(); + properties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123"); + + Connector connector = new FlussConnectorProvider().create(properties, context()); + Assertions.assertTrue(connector instanceof FlussConnector); + connector.close(); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java new file mode 100644 index 00000000000000..90468bf30c6c06 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Programmable {@link FlussAdminOps} for unit tests: canned answers plus a log of the calls made. + * + *

This is the connector's substitute for a mocking framework (which the fe-connector modules do not + * carry). Two things it buys that a live cluster cannot: a test can put the cluster in states that are + * awkward or slow to reach for real — a bucket whose lake offset has caught up with its log, a partition + * present only in the lake — and it can assert which remote calls were made, which is how + * per-statement memoization and partition pushdown are pinned down. + * + *

Anything a test has not programmed throws rather than returning a neutral value: silently answering + * "empty" would let a test pass while the code under test called something nobody meant it to call. + */ +class RecordingFlussAdminOps implements FlussAdminOps { + + /** Every call made, in order, as {@code method(args)} — the assertion target for call-count tests. */ + final List calls = new ArrayList<>(); + + List databases = Collections.emptyList(); + final Map> tablesByDatabase = new HashMap<>(); + + @Override + public List listDatabases() { + calls.add("listDatabases()"); + return databases; + } + + @Override + public boolean databaseExists(String databaseName) { + calls.add("databaseExists(" + databaseName + ")"); + return databases.contains(databaseName); + } + + @Override + public List listTables(String databaseName) { + calls.add("listTables(" + databaseName + ")"); + List tables = tablesByDatabase.get(databaseName); + if (tables == null) { + throw new IllegalStateException("no tables programmed for database '" + databaseName + "'"); + } + return tables; + } + + @Override + public boolean tableExists(TablePath tablePath) { + throw notProgrammed("tableExists"); + } + + @Override + public TableInfo getTableInfo(TablePath tablePath) { + throw notProgrammed("getTableInfo"); + } + + @Override + public List listPartitionInfos(TablePath tablePath) { + throw notProgrammed("listPartitionInfos"); + } + + @Override + public List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec) { + throw notProgrammed("listPartitionInfos"); + } + + @Override + public TableStats getTableStats(TablePath tablePath) { + throw notProgrammed("getTableStats"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath) { + throw notProgrammed("getLatestKvSnapshots"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName) { + throw notProgrammed("getLatestKvSnapshots"); + } + + @Override + public LakeSnapshot getReadableLakeSnapshot(TablePath tablePath) { + throw notProgrammed("getReadableLakeSnapshot"); + } + + @Override + public Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec) { + throw notProgrammed("listOffsets"); + } + + @Override + public Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + throw notProgrammed("listOffsets"); + } + + private static UnsupportedOperationException notProgrammed(String method) { + return new UnsupportedOperationException( + method + " was called but this test programmed no answer for it"); + } +} diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 4b3a44c05a726c..c3fd034c363bff 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -85,6 +85,7 @@ under the License. fe-connector-paimon fe-connector-hudi fe-connector-iceberg + fe-connector-fluss diff --git a/fe/pom.xml b/fe/pom.xml index 4fa658e448b4e9..280129127bd59f 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -407,6 +407,17 @@ under the License. silently breaks that FE->BE deserialization at runtime. These three MUST stay equal — do NOT override paimon.version per-module. --> 1.3.1 + + + 1.0-SNAPSHOT 3.4.4 17.0.0 From ef2f3415bb1550ddaaca8cdea3507c3adc3abe6a Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 18:42:41 +0800 Subject: [PATCH 02/35] [feat](fluss) Map fluss column types onto Doris types Adds FlussTypeMapping, a DataTypeVisitor over fluss's 19 type roots, so a type added by a future fluss release breaks this compile instead of degrading a column silently. Every rule is the composition of fluss's own FlussDataTypeToPaimonDataType with the paimon connector's PaimonTypeMapping: a datalake table is readable both as `tbl` (this mapping) and as `tbl$lake` (delegated to the paimon connector), and one table must not show two schemas. That is what fixes CHAR over 255 collapsing to STRING, the microsecond clamp on timestamps, and the spelling and defaults of the two mapping switches. TIME is marked UNSUPPORTED rather than reinterpreted: Doris has no storable TIME column, and both substitutes other engines use (STRING, elapsed-millis INT) hand back a value whose meaning differs from the source. The marker degrades one leaf, so a wide table with one TIME field still loads. MAP is mapped, not marked unsupported. Fluss reads it in the ARROW log format, in the compacted KV format behind primary-key tables, and converts it on the way into the lake, so refusing it here would only mean `tbl$lake` showing a MAP that `tbl` refuses to project. The two mapping switches reuse the unprefixed, engine-wide names the hive, paimon and iceberg catalogs already answer to; being unprefixed also keeps them out of the fluss client configuration for free. Co-Authored-By: Claude Opus 5 (1M context) --- .../fluss/FlussConnectorProperties.java | 27 ++ .../connector/fluss/FlussTypeMapping.java | 257 ++++++++++++++++++ .../fluss/FlussConnectorPropertiesTest.java | 25 ++ .../connector/fluss/FlussTypeMappingTest.java | 222 +++++++++++++++ 4 files changed, 531 insertions(+) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java index 1f391465fab25a..12df540e4de99f 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java @@ -57,6 +57,22 @@ public final class FlussConnectorProperties { */ public static final String UNION_READ_MODE = "fluss.union_read.mode"; + /** + * Optional. Whether a fluss BINARY/BYTES column reads as Doris VARBINARY instead of STRING. + * + *

Unprefixed on purpose: this is the engine-wide catalog property + * ({@code CatalogProperty.ENABLE_MAPPING_VARBINARY}) that the hive, paimon and iceberg catalogs + * already answer to, and a user should not have to learn a fluss-specific spelling for it. Being + * unprefixed also keeps it out of {@link #toFlussClientConfig} for free. + */ + public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + + /** + * Optional. Whether a fluss TIMESTAMP_LTZ column reads as Doris TIMESTAMPTZ instead of DATETIMEV2. + * Engine-wide catalog property, same reasoning as {@link #ENABLE_MAPPING_VARBINARY}. + */ + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; + /** Value set of {@link #UNION_READ_MODE}. */ public enum UnionReadMode { /** Union read when the lake has a readable snapshot, fluss-only when it does not. */ @@ -107,6 +123,17 @@ public static UnionReadMode unionReadMode(Map properties) { return value == null ? UnionReadMode.AUTO : UnionReadMode.parse(value); } + /** The type-mapping switches this catalog declares; both default to off. */ + public static FlussTypeMapping.Options typeMappingOptions(Map properties) { + return new FlussTypeMapping.Options( + booleanValue(properties, ENABLE_MAPPING_VARBINARY), + booleanValue(properties, ENABLE_MAPPING_TIMESTAMP_TZ)); + } + + private static boolean booleanValue(Map properties, String key) { + return Boolean.parseBoolean(properties.getOrDefault(key, "false")); + } + /** * The fluss client configuration this catalog describes: every {@code fluss.}-prefixed property * that is not Doris-only, with the prefix stripped. diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java new file mode 100644 index 00000000000000..e8d5d723c815e6 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorType; + +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BigIntType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.BooleanType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeVisitor; +import org.apache.fluss.types.DateType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.DoubleType; +import org.apache.fluss.types.FloatType; +import org.apache.fluss.types.IntType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.SmallIntType; +import org.apache.fluss.types.StringType; +import org.apache.fluss.types.TimeType; +import org.apache.fluss.types.TimestampType; +import org.apache.fluss.types.TinyIntType; + +import java.util.ArrayList; +import java.util.List; + +/** + * Maps a fluss {@link DataType} to a Doris {@link ConnectorType}. + * + *

Why a visitor and not a switch. Fluss's {@link DataTypeVisitor} has one method per type, + * so a type added by a future fluss release breaks this compile instead of falling into a + * {@code default:} branch and degrading a column silently. + * + *

The parity invariant. A datalake-enabled fluss table is reachable through two doors: the + * table itself (this mapping) and its lake sibling {@code tbl$lake}, which is served by the paimon + * connector over the paimon table fluss's tiering service writes. Both doors must describe the same + * column the same way, so every rule below is the composition of fluss's own + * {@code FlussDataTypeToPaimonDataType} with the paimon connector's {@code PaimonTypeMapping} — + * including the ones that look arbitrary in isolation (CHAR over 255 collapsing to STRING, the + * microsecond clamp, the two mapping switches). Changing a rule here without checking that + * composition makes one table show two schemas. + * + *

Unsupported types degrade, they do not throw. A type Doris cannot represent becomes the + * {@code UNSUPPORTED} marker, which fe-core turns into {@code Type.UNSUPPORTED}: the table still + * loads and still answers queries over its other columns, and only a query that projects that column + * fails. Throwing here would make one exotic column cost the whole table. + */ +public final class FlussTypeMapping implements DataTypeVisitor { + + /** Doris DATETIMEV2 stops at microseconds; fluss timestamps go to nanoseconds (precision 9). */ + private static final int MAX_TIMESTAMP_SCALE = 6; + + /** Doris CHAR stops at 255 characters; a longer fluss CHAR becomes STRING. */ + private static final int MAX_CHAR_LENGTH = 255; + + /** The type name fe-core's converter turns into {@code Type.UNSUPPORTED}. */ + private static final String UNSUPPORTED = "UNSUPPORTED"; + + private final Options options; + + private FlussTypeMapping(Options options) { + this.options = options; + } + + /** Convert one fluss type, top level or nested, under the given catalog mapping switches. */ + public static ConnectorType toConnectorType(DataType dataType, Options options) { + return dataType.accept(new FlussTypeMapping(options)); + } + + @Override + public ConnectorType visit(CharType charType) { + int length = charType.getLength(); + if (length > MAX_CHAR_LENGTH) { + return ConnectorType.of("STRING"); + } + return ConnectorType.of("CHAR", length, 0); + } + + @Override + public ConnectorType visit(StringType stringType) { + // Fluss STRING carries no declared length, so there is no VARCHAR(n) to preserve. + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(BooleanType booleanType) { + return ConnectorType.of("BOOLEAN"); + } + + @Override + public ConnectorType visit(BinaryType binaryType) { + if (options.isMapBinaryToVarbinary()) { + // Doris has no fixed-length binary; the declared length survives as the VARBINARY bound. + return ConnectorType.of("VARBINARY", binaryType.getLength(), 0); + } + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(BytesType bytesType) { + if (options.isMapBinaryToVarbinary()) { + // Unbounded in fluss: leave the length unset so fe-core's converter fills in the Doris + // VARBINARY maximum, which is what the paimon side (BYTES = VARBINARY(Integer.MAX_VALUE)) + // resolves to as well. + return ConnectorType.of("VARBINARY"); + } + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(DecimalType decimalType) { + return ConnectorType.of("DECIMALV3", decimalType.getPrecision(), decimalType.getScale()); + } + + @Override + public ConnectorType visit(TinyIntType tinyIntType) { + return ConnectorType.of("TINYINT"); + } + + @Override + public ConnectorType visit(SmallIntType smallIntType) { + return ConnectorType.of("SMALLINT"); + } + + @Override + public ConnectorType visit(IntType intType) { + return ConnectorType.of("INT"); + } + + @Override + public ConnectorType visit(BigIntType bigIntType) { + return ConnectorType.of("BIGINT"); + } + + @Override + public ConnectorType visit(FloatType floatType) { + return ConnectorType.of("FLOAT"); + } + + @Override + public ConnectorType visit(DoubleType doubleType) { + return ConnectorType.of("DOUBLE"); + } + + @Override + public ConnectorType visit(DateType dateType) { + return ConnectorType.of("DATEV2"); + } + + @Override + public ConnectorType visit(TimeType timeType) { + // Doris has no storable TIME column type. Mapping it to STRING or to the elapsed-millis INT + // that other engines use would hand back a value whose meaning differs from the source, so the + // column is marked unsupported instead. The paimon and iceberg connectors mark their own TIME + // the same way, which also keeps tbl and tbl$lake agreeing. + return ConnectorType.of(UNSUPPORTED); + } + + @Override + public ConnectorType visit(TimestampType timestampType) { + return ConnectorType.of("DATETIMEV2", clampScale(timestampType.getPrecision()), 0); + } + + @Override + public ConnectorType visit(LocalZonedTimestampType localZonedTimestampType) { + int scale = clampScale(localZonedTimestampType.getPrecision()); + if (options.isMapTimestampTz()) { + return ConnectorType.of("TIMESTAMPTZ", scale, 0); + } + return ConnectorType.of("DATETIMEV2", scale, 0); + } + + @Override + public ConnectorType visit(ArrayType arrayType) { + // Element nullability is deliberately not carried: the paimon connector's read path leaves it + // at the default, and the two must not disagree (see the parity invariant). + return ConnectorType.arrayOf(arrayType.getElementType().accept(this)); + } + + @Override + public ConnectorType visit(MapType mapType) { + // MAP is a first-class fluss type on every path Doris uses: the ARROW log format, the compacted + // KV format behind primary-key tables, and the paimon table the tiering service writes. Marking + // it unsupported here would leave tbl$lake showing a MAP that tbl refuses to project. + return ConnectorType.mapOf( + mapType.getKeyType().accept(this), + mapType.getValueType().accept(this)); + } + + @Override + public ConnectorType visit(RowType rowType) { + List fields = rowType.getFields(); + List names = new ArrayList<>(fields.size()); + List types = new ArrayList<>(fields.size()); + List nullable = new ArrayList<>(fields.size()); + List comments = new ArrayList<>(fields.size()); + for (DataField field : fields) { + names.add(field.getName()); + types.add(field.getType().accept(this)); + // Nested NOT NULL and COMMENT are carried so DESCRIBE / SHOW CREATE TABLE report what the + // fluss schema actually declares for the field. + nullable.add(field.getType().isNullable()); + comments.add(field.getDescription().orElse(null)); + } + return ConnectorType.structOf(names, types, nullable, comments); + } + + private static int clampScale(int precision) { + return Math.min(precision, MAX_TIMESTAMP_SCALE); + } + + /** + * The catalog-level switches that change a mapping. Both default to off and both carry the same + * name and meaning they have on other Doris catalogs (see {@link FlussConnectorProperties}). + */ + public static final class Options { + + public static final Options DEFAULT = new Options(false, false); + + private final boolean mapBinaryToVarbinary; + private final boolean mapTimestampTz; + + public Options(boolean mapBinaryToVarbinary, boolean mapTimestampTz) { + this.mapBinaryToVarbinary = mapBinaryToVarbinary; + this.mapTimestampTz = mapTimestampTz; + } + + public boolean isMapBinaryToVarbinary() { + return mapBinaryToVarbinary; + } + + public boolean isMapTimestampTz() { + return mapTimestampTz; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java index b70c862da47531..a9aa8749c55f95 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java @@ -101,6 +101,8 @@ public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() { FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", "fluss.client.security.protocol", "sasl", FlussConnectorProperties.UNION_READ_MODE, "required", + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY, "true", + FlussConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true", "type", "fluss", "warehouse", "s3://ignored"); @@ -110,9 +112,32 @@ public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() { // switch and every non-fluss catalog property stay behind. The engine's own keys ("type") and // other connectors' keys ("warehouse") are not fluss options and must not be handed over as if // they were — the fluss config is not a place to dump whatever the catalog happened to carry. + // The two type-mapping switches are in the input for the same reason: they steer Doris's own + // schema rendering and fluss has no idea what they mean. Map expected = new HashMap<>(); expected.put("bootstrap.servers", "localhost:9123"); expected.put("client.security.protocol", "sasl"); Assertions.assertEquals(expected, config); } + + @Test + public void typeMappingSwitchesDefaultToOffAndUseTheEngineWideNames() { + // The names are deliberately the unprefixed, engine-wide ones the hive/paimon/iceberg catalogs + // already answer to: a user who knows enable.mapping.varbinary must not have to discover a + // fluss-specific spelling, and a misspelling here degrades silently to "switch is off". + FlussTypeMapping.Options off = FlussConnectorProperties.typeMappingOptions(props()); + Assertions.assertFalse(off.isMapBinaryToVarbinary()); + Assertions.assertFalse(off.isMapTimestampTz()); + + FlussTypeMapping.Options on = FlussConnectorProperties.typeMappingOptions(props( + "enable.mapping.varbinary", "true", + "enable.mapping.timestamp_tz", "TRUE")); + Assertions.assertTrue(on.isMapBinaryToVarbinary()); + Assertions.assertTrue(on.isMapTimestampTz()); + + // Anything that is not "true" is off, matching how every other catalog reads these. + FlussTypeMapping.Options garbage = FlussConnectorProperties.typeMappingOptions(props( + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY, "yes")); + Assertions.assertFalse(garbage.isMapBinaryToVarbinary()); + } } diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java new file mode 100644 index 00000000000000..545131e2199de0 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorType; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Pins what a fluss column looks like once Doris describes it. + * + *

Two properties are worth more than the individual rows. The first is completeness: a fluss + * release that adds a type must not slip through as a silently degraded column, so + * {@link #everyFlussTypeRootIsMapped} fails the build when a {@link DataTypeRoot} has no case here. + * The second is that these rules are not free choices — they are the composition of fluss's own + * fluss-to-paimon conversion with the paimon connector's mapping, because a datalake table is + * readable both as {@code tbl} (this mapping) and as {@code tbl$lake} (the paimon connector), and one + * table must not show two schemas. + */ +public class FlussTypeMappingTest { + + private static ConnectorType map(DataType type) { + return FlussTypeMapping.toConnectorType(type, FlussTypeMapping.Options.DEFAULT); + } + + private static ConnectorType map(DataType type, boolean varbinary, boolean timestampTz) { + return FlussTypeMapping.toConnectorType(type, new FlussTypeMapping.Options(varbinary, timestampTz)); + } + + @Test + public void everyFlussTypeRootIsMapped() { + Map expected = new LinkedHashMap<>(); + expected.put(DataTypes.CHAR(10), ConnectorType.of("CHAR", 10, 0)); + expected.put(DataTypes.STRING(), ConnectorType.of("STRING")); + expected.put(DataTypes.BOOLEAN(), ConnectorType.of("BOOLEAN")); + expected.put(DataTypes.BINARY(16), ConnectorType.of("STRING")); + expected.put(DataTypes.BYTES(), ConnectorType.of("STRING")); + expected.put(DataTypes.DECIMAL(20, 4), ConnectorType.of("DECIMALV3", 20, 4)); + expected.put(DataTypes.TINYINT(), ConnectorType.of("TINYINT")); + expected.put(DataTypes.SMALLINT(), ConnectorType.of("SMALLINT")); + expected.put(DataTypes.INT(), ConnectorType.of("INT")); + expected.put(DataTypes.BIGINT(), ConnectorType.of("BIGINT")); + expected.put(DataTypes.FLOAT(), ConnectorType.of("FLOAT")); + expected.put(DataTypes.DOUBLE(), ConnectorType.of("DOUBLE")); + expected.put(DataTypes.DATE(), ConnectorType.of("DATEV2")); + expected.put(DataTypes.TIME(3), ConnectorType.of("UNSUPPORTED")); + expected.put(DataTypes.TIMESTAMP(6), ConnectorType.of("DATETIMEV2", 6, 0)); + expected.put(DataTypes.TIMESTAMP_LTZ(6), ConnectorType.of("DATETIMEV2", 6, 0)); + expected.put(DataTypes.ARRAY(DataTypes.INT()), ConnectorType.arrayOf(ConnectorType.of("INT"))); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + ConnectorType.mapOf(ConnectorType.of("STRING"), ConnectorType.of("INT"))); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), + ConnectorType.structOf(singleton("f"), singleton(ConnectorType.of("INT")))); + + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : expected.entrySet()) { + covered.add(entry.getKey().getTypeRoot()); + Assertions.assertEquals(entry.getValue(), map(entry.getKey()), + "mapping of fluss " + entry.getKey()); + } + // The gate: fluss owns this enum, so a new root appearing here means fluss grew a type that + // nobody has decided how to show in Doris yet. Decide it, then add the row above. + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "every fluss type root needs a case; missing: " + + complement(covered)); + } + + @Test + public void charLongerThanTheDorisMaximumBecomesString() { + // Doris CHAR stops at 255. Truncating would lose data and failing would make the table + // unloadable, so the column widens to STRING - the same relief valve the paimon connector uses, + // which is what keeps tbl and tbl$lake agreeing on a CHAR(1000) column. + Assertions.assertEquals(ConnectorType.of("CHAR", 255, 0), map(DataTypes.CHAR(255))); + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.CHAR(256))); + } + + @Test + public void timestampPrecisionIsClampedToMicroseconds() { + // Fluss keeps up to nanoseconds, Doris DATETIMEV2 up to microseconds. The clamp must be a clamp + // and not an error: a nanosecond column still reads, it just reads with microsecond scale. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 0, 0), map(DataTypes.TIMESTAMP(0))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), map(DataTypes.TIMESTAMP(3))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), map(DataTypes.TIMESTAMP(9))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), map(DataTypes.TIMESTAMP_LTZ(9))); + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 6, 0), + map(DataTypes.TIMESTAMP_LTZ(9), false, true)); + } + + @Test + public void timeIsMarkedUnsupportedRatherThanReinterpreted() { + // Doris has no storable TIME column. The two tempting substitutes both lie about the value - + // STRING changes its type and the elapsed-millis INT other engines use changes its meaning - so + // the column is marked unsupported and only a query that projects it fails. + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), map(DataTypes.TIME())); + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), map(DataTypes.TIME(3))); + } + + @Test + public void anUnsupportedLeafDoesNotSinkTheColumnsAroundIt() { + // The marker degrades exactly one leaf. A wide table with one TIME field inside a struct still + // loads, and its neighbouring fields stay readable - that is the whole reason this is a marker + // and not an exception. + ConnectorType struct = map(DataTypes.ROW( + DataTypes.FIELD("started", DataTypes.TIME(3)), + DataTypes.FIELD("id", DataTypes.BIGINT()))); + + Assertions.assertEquals("STRUCT", struct.getTypeName()); + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), struct.getChildren().get(0)); + Assertions.assertEquals(ConnectorType.of("BIGINT"), struct.getChildren().get(1)); + } + + @Test + public void theBinaryFamilyFollowsTheVarbinarySwitch() { + // Off by default (STRING), because that is what every other Doris catalog does with binary + // columns and flipping the default would change what existing queries return. + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.BINARY(16))); + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.BYTES())); + + // On: fixed-length BINARY(n) keeps n as the VARBINARY bound; unbounded BYTES declares no length + // so fe-core fills in the Doris VARBINARY maximum. + Assertions.assertEquals(ConnectorType.of("VARBINARY", 16, 0), map(DataTypes.BINARY(16), true, false)); + Assertions.assertEquals(ConnectorType.of("VARBINARY"), map(DataTypes.BYTES(), true, false)); + } + + @Test + public void timestampLtzFollowsTheTimestampTzSwitch() { + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), map(DataTypes.TIMESTAMP_LTZ(3))); + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 3, 0), + map(DataTypes.TIMESTAMP_LTZ(3), false, true)); + + // The two switches are independent: neither reads the other's property. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), + map(DataTypes.TIMESTAMP_LTZ(3), true, false)); + Assertions.assertEquals(ConnectorType.of("VARBINARY", 4, 0), + map(DataTypes.BINARY(4), true, true)); + } + + @Test + public void nestedTypesRecurseWithTheSameRulesAndSwitches() { + // ARRAY> and MAP>: the rules that apply to a top-level column apply + // at any depth, switches included - a BINARY buried three levels down must not quietly ignore + // enable.mapping.varbinary while the top-level one honours it. + ConnectorType arrayOfStruct = map(DataTypes.ARRAY(DataTypes.ROW( + DataTypes.FIELD("k", DataTypes.CHAR(300)), + DataTypes.FIELD("v", DataTypes.BINARY(8)))), true, false); + Assertions.assertEquals( + ConnectorType.arrayOf(ConnectorType.structOf( + pair("k", "v"), + pair(ConnectorType.of("STRING"), ConnectorType.of("VARBINARY", 8, 0)))), + arrayOfStruct); + + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("STRING"), + ConnectorType.arrayOf(ConnectorType.of("INT"))), + map(DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT())))); + + // MAP is mapped, not marked unsupported: fluss reads it in the ARROW log format and in the + // compacted KV format, and the lake sibling served by the paimon connector shows it as a MAP, + // so refusing it here would give one table two different schemas. + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("INT"), + ConnectorType.structOf(singleton("n"), singleton(ConnectorType.of("DOUBLE")))), + map(DataTypes.MAP(DataTypes.INT(), + DataTypes.ROW(DataTypes.FIELD("n", DataTypes.DOUBLE()))))); + } + + @Test + public void structFieldsKeepTheirDeclaredNullabilityAndComment() { + // These two never affect the type's identity, only what DESCRIBE and SHOW CREATE TABLE print. + // Dropping them would silently report every nested field as a nullable, undocumented column. + ConnectorType struct = map(DataTypes.ROW( + DataTypes.FIELD("required", DataTypes.INT().copy(false), "the id"), + DataTypes.FIELD("optional", DataTypes.STRING()))); + + Assertions.assertEquals(pair("required", "optional"), struct.getFieldNames()); + Assertions.assertFalse(struct.isChildNullable(0), "a NOT NULL fluss field must not read as nullable"); + Assertions.assertEquals("the id", struct.getChildComment(0)); + Assertions.assertTrue(struct.isChildNullable(1)); + Assertions.assertNull(struct.getChildComment(1), "a field with no description has no comment"); + } + + private static Set complement(Set covered) { + Set missing = EnumSet.allOf(DataTypeRoot.class); + missing.removeAll(covered); + return missing; + } + + private static List singleton(T value) { + return Collections.singletonList(value); + } + + private static List pair(T first, T second) { + return Arrays.asList(first, second); + } +} From f48edfb87340d4b9d1a6a5c34bdd367887676c20 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 19:12:51 +0800 Subject: [PATCH 03/35] [feat](fluss) Describe a fluss table through the connector metadata SPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the metadata surface: the table handle, the Doris schema and column handles, partition listing, statistics and the table descriptor. The handle snapshots what split planning will need — primary key, bucket count and keys, partition keys, lake format, and the fluss table property map, which for a datalake table is where the fluss coordinator merges the lake catalog's own connection settings (the Doris catalog is configured with bootstrap servers and nothing else). Taking them one at a time later could straddle an ALTER; taking them together cannot. Column comments are read from the fluss schema, not from the table's row type: fluss rebuilds that row type from the schema and drops every field description on the way, so reading the row type would report every column as undocumented. Columns are reported nullable throughout even though fluss marks primary-key columns NOT NULL — propagating it would let the planner fold null-rejecting predicates on this path while the same table read through its lake sibling (the paimon connector, which reports everything nullable) keeps them. Partitions are rendered in the Hive-style `k=v/k=v` naming fe-core parses back, not fluss's own `v$v`. No escaping is needed: fluss rejects any partition value outside ASCII alphanumerics, `_` and `-`, so no value can contain the separators, and none can be SQL NULL. One statement reads a table once. The handle, the schema, the column handles and the comment share a per-statement memo, which beyond the round trips is what stops a concurrent ALTER from being visible to half of one plan. Tests come in two layers. The unit layer drives the awkward states a cluster will not produce on request — a table that vanished, a coordinator that is down, a partition spec whose map iterates in a different order than the partition columns. Over it, FlussMetadataClusterTest starts a real coordinator, tablet server and ZooKeeper in the test JVM and reads tables created through the fluss client, which is the only way to check the premises the unit layer assumes. It is named ...Test rather than ...ITCase because surefire's default includes do not match *ITCase, and that name would leave the class unexecuted under a green build. Co-Authored-By: Claude Opus 5 (1M context) --- fe/fe-connector/fe-connector-fluss/pom.xml | 56 +++ .../connector/fluss/FlussColumnHandle.java | 73 ++++ .../doris/connector/fluss/FlussConnector.java | 3 +- .../fluss/FlussConnectorMetadata.java | 234 ++++++++++- .../connector/fluss/FlussStatementScope.java | 58 +++ .../connector/fluss/FlussTableHandle.java | 204 ++++++++++ .../fluss/FlussConnectorMetadataTest.java | 375 +++++++++++++++++- .../fluss/FlussMetadataClusterTest.java | 287 ++++++++++++++ .../fluss/FlussStatementScopeTest.java | 53 +++ .../connector/fluss/FlussTableHandleTest.java | 96 +++++ .../connector/fluss/FlussTestSession.java | 96 +++++ .../connector/fluss/FlussTestTables.java | 114 ++++++ .../fluss/RecordingFlussAdminOps.java | 34 +- 13 files changed, 1676 insertions(+), 7 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml index 0633d1e1b0fdc4..168c64cec8efb8 100644 --- a/fe/fe-connector/fe-connector-fluss/pom.xml +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -75,11 +75,67 @@ under the License. ${fluss.version} + + + ${project.groupId} + fe-thrift + ${project.version} + provided + + + + org.apache.logging.log4j + log4j-api + + org.junit.jupiter junit-jupiter test + + + + org.apache.fluss + fluss-server + ${fluss.version} + test + + + + org.apache.fluss + fluss-server + ${fluss.version} + test-jar + test + + + + + org.apache.fluss + fluss-test-utils + ${fluss.version} + test + + + + + org.apache.curator + curator-test + 5.4.0 + test + diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java new file mode 100644 index 00000000000000..d054eed5ad948f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; + +import java.util.Objects; + +/** + * A fluss column: its name and its position in the table's row type. + * + *

The position is what the scan side needs — fluss projects by field index + * ({@code TableScan.project(int[])}) and its readers hand back positional rows — so carrying it here + * keeps the projection from having to re-derive it from the schema. + */ +public class FlussColumnHandle implements ConnectorColumnHandle { + + private static final long serialVersionUID = 1L; + + private final String name; + private final int fieldIndex; + + public FlussColumnHandle(String name, int fieldIndex) { + this.name = Objects.requireNonNull(name, "name"); + this.fieldIndex = fieldIndex; + } + + public String getName() { + return name; + } + + public int getFieldIndex() { + return fieldIndex; + } + + /** Identity is the name and the position: the same name at a different index is a different column. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FlussColumnHandle)) { + return false; + } + FlussColumnHandle that = (FlussColumnHandle) o; + return fieldIndex == that.fieldIndex && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, fieldIndex); + } + + @Override + public String toString() { + return name + "[" + fieldIndex + "]"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 5d79a533472a9b..02aa7c7c10cf38 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -57,7 +57,8 @@ public FlussConnector(Map properties, ConnectorContext context) @Override public ConnectorMetadata getMetadata(ConnectorSession session) { - return new FlussConnectorMetadata(adminOps()); + return new FlussConnectorMetadata(adminOps(), + FlussConnectorProperties.typeMappingOptions(properties)); } @Override diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 6a050683b7a401..3b37775477f075 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -17,23 +17,60 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.api.ConnectorColumn; import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Fluss metadata for one statement: a thin mapping from the connector SPI onto {@link FlussAdminOps}. * *

Fluss has a real two-level namespace (database, table), so the listing calls are direct * pass-throughs and carry no Doris-side naming convention. + * + *

Every method that needs the table's schema goes through {@link #tableInfo}, which memoizes the + * fetch for the statement — the handle, the schema, the column handles and (later) split planning + * therefore see one coherent version of the table for one round trip. */ public class FlussConnectorMetadata implements ConnectorMetadata { + private static final Logger LOG = LogManager.getLogger(FlussConnectorMetadata.class); + + /** What {@code getTableSchema} reports as the table's format; surfaces in DESCRIBE / EXPLAIN. */ + private static final String TABLE_FORMAT_TYPE = "FLUSS"; + private final FlussAdminOps adminOps; + private final FlussTypeMapping.Options typeMappingOptions; - public FlussConnectorMetadata(FlussAdminOps adminOps) { + public FlussConnectorMetadata(FlussAdminOps adminOps, FlussTypeMapping.Options typeMappingOptions) { this.adminOps = adminOps; + this.typeMappingOptions = typeMappingOptions; } @Override @@ -50,4 +87,199 @@ public boolean databaseExists(ConnectorSession session, String dbName) { public List listTableNames(ConnectorSession session, String dbName) { return adminOps.listTables(dbName); } + + /** + * Resolves a table, or reports that it does not exist. + * + *

Only fluss's own "not there" exceptions become an empty handle. Anything else — an + * unreachable coordinator, a timeout, an auth failure — propagates, because answering "no such + * table" to those makes a broken catalog look like an empty one: {@code SELECT} then fails with + * "table not found", a {@code CREATE TABLE IF NOT EXISTS} would look free to proceed, and the real + * error never reaches the user. + */ + @Override + public Optional getTableHandle( + ConnectorSession session, String dbName, String tableName) { + try { + return Optional.of(FlussTableHandle.of(tableInfo(session, TablePath.of(dbName, tableName)))); + } catch (TableNotExistException | DatabaseNotExistException e) { + return Optional.empty(); + } + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + TableInfo info = tableInfo(session, flussHandle.toTablePath()); + + List schemaColumns = info.getSchema().getColumns(); + List columns = new ArrayList<>(schemaColumns.size()); + for (Schema.Column column : schemaColumns) { + columns.add(toConnectorColumn(column)); + } + + // LinkedHashMap: SHOW CREATE TABLE renders PROPERTIES from this map, and a stable order keeps + // the rendered DDL from churning between runs. + Map properties = new LinkedHashMap<>(info.getProperties().toMap()); + if (flussHandle.isPartitioned()) { + // "partition_columns" is the key the generic fe-core consumer reads; without it the table is + // treated as unpartitioned and partition pruning is silently lost. Names are case-preserved + // to stay matchable against the column names emitted above. + properties.put(ConnectorTableSchema.PARTITION_COLUMNS_KEY, + String.join(",", flussHandle.getPartitionKeys())); + } + return new ConnectorTableSchema( + flussHandle.getTableName(), columns, TABLE_FORMAT_TYPE, properties); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + List columns = tableInfo(session, flussHandle.toTablePath()).getSchema().getColumns(); + Map handles = new LinkedHashMap<>(); + for (int i = 0; i < columns.size(); i++) { + String name = columns.get(i).getName(); + handles.put(name, new FlussColumnHandle(name, i)); + } + return handles; + } + + @Override + public String getTableComment(ConnectorSession session, String dbName, String tableName) { + return tableInfo(session, TablePath.of(dbName, tableName)).getComment().orElse(""); + } + + @Override + public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + List partitions = listPartitions(session, handle, Optional.empty()); + List names = new ArrayList<>(partitions.size()); + for (ConnectorPartitionInfo partition : partitions) { + names.add(partition.getPartitionName()); + } + return names; + } + + /** + * Lists the table's partitions in the Hive-style {@code k1=v1/k2=v2} naming every Doris catalog + * uses, which is not fluss's own {@code v1$v2} spelling — fe-core parses the segments back out of + * the name for {@code SHOW PARTITIONS} and the {@code partition_values} function. No escaping is + * needed on the way: fluss rejects a partition value that is not ASCII alphanumerics, {@code _} or + * {@code -} (TablePath#detectInvalidName, applied by PartitionUtils#validatePartitionValues), so a + * value can contain neither the {@code =} nor the {@code /} that would make two partitions render + * to one name. That same rule is why no value can be SQL NULL and the null-flag list stays empty. + * + *

{@code filter} is ignored: server-side partition pruning exists in fluss (a partial + * {@code PartitionSpec}) but it takes a spec, not a predicate, and the predicate-to-spec reduction + * belongs with split planning, which is the caller that has a predicate worth pushing. Listing + * returns everything, as the paimon and maxcompute connectors do. + */ + @Override + public List listPartitions(ConnectorSession session, + ConnectorTableHandle handle, Optional filter) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + List partitionKeys = flussHandle.getPartitionKeys(); + if (partitionKeys.isEmpty()) { + // Guard before the remote call: asking fluss for the partitions of an unpartitioned table is + // an error there, and "this table has no partitions" is already known from the handle. + return Collections.emptyList(); + } + + List flussPartitions = adminOps.listPartitionInfos(flussHandle.toTablePath()); + List result = new ArrayList<>(flussPartitions.size()); + for (PartitionInfo partition : flussPartitions) { + Map spec = partition.getPartitionSpec().getSpecMap(); + // Both lists follow the partition-COLUMN order, not the spec's iteration order, because + // fe-core zips them positionally against the partition columns. + Map values = new LinkedHashMap<>(); + List orderedValues = new ArrayList<>(partitionKeys.size()); + StringBuilder name = new StringBuilder(); + for (String partitionKey : partitionKeys) { + String value = spec.get(partitionKey); + values.put(partitionKey, value); + orderedValues.add(value); + if (name.length() > 0) { + name.append('/'); + } + name.append(partitionKey).append('=').append(value); + } + result.add(new ConnectorPartitionInfo( + name.toString(), values, Collections.emptyMap(), + orderedValues, Collections.emptyList())); + } + return result; + } + + /** + * The table's row count, when fluss has one. + * + *

Fluss reports row count only (no data size, no per-column statistics), and only for a table + * whose statistics are enabled — otherwise the count comes back as zero, which is reported as + * unknown rather than as "the table is empty". Statistics are best effort by contract: a failure + * degrades to unknown instead of failing the statement, because this runs in background analysis + * and in SHOW, where a transient coordinator error must not surface as a query error. + */ + @Override + public Optional getTableStatistics( + ConnectorSession session, ConnectorTableHandle handle) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + long rowCount; + try { + rowCount = adminOps.getTableStats(flussHandle.toTablePath()).getRowCount(); + } catch (Exception e) { + LOG.warn("Failed to read fluss table statistics for {}", flussHandle, e); + return Optional.empty(); + } + if (rowCount <= 0) { + return Optional.empty(); + } + // -1 = unknown data size. Zero would tell the optimizer the table costs nothing to scan. + return Optional.of(new ConnectorTableStatistics(rowCount, -1)); + } + + /** + * The Thrift table descriptor the BE receives. A fluss scan reaches the BE through the same file + * scan node the lake connectors use (the fluss ranges ride in its format-specific descriptor), so + * the table descriptor is the generic hive-shaped one those connectors send, exactly as paimon and + * hudi do; a fluss-specific Thrift table type would buy nothing the scan path reads. + */ + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, + long tableId, String tableName, String dbName, + String remoteName, int numCols, long catalogId) { + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); + descriptor.setHiveTable(new THiveTable(dbName, tableName, new LinkedHashMap<>())); + return descriptor; + } + + private ConnectorColumn toConnectorColumn(Schema.Column column) { + ConnectorType type = FlussTypeMapping.toConnectorType(column.getDataType(), typeMappingOptions); + // isKey=true for every column and nullable=true for every column: this is what every Doris + // external catalog reports. The nullability one matters beyond convention — fluss marks its + // primary-key columns NOT NULL, and propagating that would let the planner fold null-rejecting + // predicates, while the same table read through its lake sibling (the paimon connector, which + // reports every column nullable) would keep them. One table must not get two different plans + // depending on which door it was read through. + ConnectorColumn connectorColumn = new ConnectorColumn( + column.getName(), + type, + column.getComment().orElse(""), + true, + null, + true); + // A "with local time zone" timestamp carries the WITH_TIMEZONE marker DESCRIBE shows in Extra. + // Keyed on the SOURCE fluss type, so it survives whether enable.mapping.timestamp_tz mapped the + // column to TIMESTAMPTZ or to plain DATETIME. + if (column.getDataType().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + connectorColumn = connectorColumn.withTimeZone(); + } + return connectorColumn; + } + + /** The table's metadata, fetched once per statement (see {@link FlussStatementScope}). */ + private TableInfo tableInfo(ConnectorSession session, TablePath tablePath) { + return FlussStatementScope.sharedTableInfo(session, tablePath, + () -> adminOps.getTableInfo(tablePath)); + } } diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java new file mode 100644 index 00000000000000..6d0c09839019ca --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScopes; + +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import java.util.function.Supplier; + +/** + * Per-statement sharing of one table's {@link TableInfo}. + * + *

A single statement asks for the same table's metadata several times over — the handle, the + * schema, the column handles, then split planning — and each of those is a coordinator round trip on + * its own. Routing them through the statement scope collapses them to one fetch, and it also makes the + * statement self-consistent: without it, a concurrent ALTER could land between two of those calls and + * leave the plan built from two different schema versions. + * + *

Under a {@code null} session or a statement scope of {@code NONE} (offline, no live statement) + * the loader simply runs every time, which is what an untracked call did before. + */ +final class FlussStatementScope { + + /** + * Namespace for fluss's per-statement {@link TableInfo} memo. Prefixed with this connector's type + * name ("fluss") per the {@link ConnectorStatementScopes} convention, so a gateway statement + * spanning two connectors cannot hand one of them the other's value. + */ + static final String TABLE_INFO_NAMESPACE = "fluss.table_info"; + + private FlussStatementScope() { + } + + static TableInfo sharedTableInfo(ConnectorSession session, TablePath tablePath, + Supplier loader) { + return ConnectorStatementScopes.resolveInStatement( + session, TABLE_INFO_NAMESPACE, + tablePath.getDatabaseName(), tablePath.getTableName(), loader); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java new file mode 100644 index 00000000000000..007319ec4293ae --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.apache.fluss.metadata.DataLakeFormat; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A fluss table, as everything downstream of {@code getTableHandle} needs to know about it. + * + *

The fields are the ones split planning reads, snapshotted from the {@link TableInfo} the handle + * was built from: whether the table has a primary key (which of the two scanner families reads it), + * the bucket count and keys (how many splits and how they line up with the lake's), the partition keys, + * and whether the table is tiered into a lake at all. They are copied rather than looked up again + * because planning must see one coherent view of the table, not a field from before an ALTER and a + * field from after it. + * + *

{@link #getProperties()} is the fluss table's own property map, and it carries more than the user + * wrote: for a datalake-enabled table the fluss coordinator merges the cluster's lake-catalog + * connection settings into it under {@code table.datalake..*}, which is where the lake side of + * a union read gets its catalog configuration. Nothing else in Doris supplies it — the Doris catalog is + * configured with fluss bootstrap servers only. + * + *

The column schema is deliberately NOT here. It is the one part that a statement re-reads through + * {@link FlussStatementScope}, so the handle stays a small, serializable identity object. + */ +public class FlussTableHandle implements ConnectorTableHandle { + + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String tableName; + private final long tableId; + private final int schemaId; + private final boolean hasPrimaryKey; + private final List primaryKeys; + private final List bucketKeys; + private final int bucketCount; + private final List partitionKeys; + private final boolean dataLakeEnabled; + /** The lake format's fluss name ({@code "paimon"}), or {@code null} when the table declares none. */ + private final String dataLakeFormat; + private final Map properties; + + public FlussTableHandle(String databaseName, String tableName, long tableId, int schemaId, + boolean hasPrimaryKey, List primaryKeys, List bucketKeys, int bucketCount, + List partitionKeys, boolean dataLakeEnabled, String dataLakeFormat, + Map properties) { + this.databaseName = Objects.requireNonNull(databaseName, "databaseName"); + this.tableName = Objects.requireNonNull(tableName, "tableName"); + this.tableId = tableId; + this.schemaId = schemaId; + this.hasPrimaryKey = hasPrimaryKey; + this.primaryKeys = copyOf(primaryKeys); + this.bucketKeys = copyOf(bucketKeys); + this.bucketCount = bucketCount; + this.partitionKeys = copyOf(partitionKeys); + this.dataLakeEnabled = dataLakeEnabled; + this.dataLakeFormat = dataLakeFormat; + this.properties = properties == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + /** Snapshots {@code tableInfo} into a handle. */ + public static FlussTableHandle of(TableInfo tableInfo) { + TablePath path = tableInfo.getTablePath(); + DataLakeFormat lakeFormat = tableInfo.getTableConfig().getDataLakeFormat().orElse(null); + return new FlussTableHandle( + path.getDatabaseName(), + path.getTableName(), + tableInfo.getTableId(), + tableInfo.getSchemaId(), + tableInfo.hasPrimaryKey(), + tableInfo.getPrimaryKeys(), + tableInfo.getBucketKeys(), + tableInfo.getNumBuckets(), + tableInfo.getPartitionKeys(), + tableInfo.getTableConfig().isDataLakeEnabled(), + lakeFormat == null ? null : lakeFormat.toString(), + tableInfo.getProperties().toMap()); + } + + public TablePath toTablePath() { + return TablePath.of(databaseName, tableName); + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public long getTableId() { + return tableId; + } + + public int getSchemaId() { + return schemaId; + } + + public boolean hasPrimaryKey() { + return hasPrimaryKey; + } + + public List getPrimaryKeys() { + return primaryKeys; + } + + public List getBucketKeys() { + return bucketKeys; + } + + public int getBucketCount() { + return bucketCount; + } + + public List getPartitionKeys() { + return partitionKeys; + } + + public boolean isPartitioned() { + return !partitionKeys.isEmpty(); + } + + public boolean isDataLakeEnabled() { + return dataLakeEnabled; + } + + public String getDataLakeFormat() { + return dataLakeFormat; + } + + public Map getProperties() { + return properties; + } + + /** + * Identity is the table plus the schema version it was read at: two handles for the same table at + * different schema versions describe different column sets and must not compare equal. The + * remaining fields all derive from that pair. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FlussTableHandle)) { + return false; + } + FlussTableHandle that = (FlussTableHandle) o; + return tableId == that.tableId + && schemaId == that.schemaId + && databaseName.equals(that.databaseName) + && tableName.equals(that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName, tableId, schemaId); + } + + @Override + public String toString() { + return "FlussTableHandle{" + databaseName + "." + tableName + + ", tableId=" + tableId + ", schemaId=" + schemaId + + ", primaryKey=" + hasPrimaryKey + ", buckets=" + bucketCount + + ", partitionKeys=" + partitionKeys + + ", dataLake=" + (dataLakeEnabled ? dataLakeFormat : "disabled") + "}"; + } + + private static List copyOf(List values) { + return values == null || values.isEmpty() + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(values)); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java index dbaee1b154fe6f..a74c536695c0e6 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java @@ -17,17 +17,83 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; +import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** - * Covers the namespace listing surface of the fluss metadata. + * Covers the metadata surface a fluss table is described through: the handle planning later reads, + * the Doris schema DESCRIBE shows, the partition list pruning depends on, and the statistics. + * + *

Everything here runs against {@link RecordingFlussAdminOps} rather than a cluster, which is what + * makes the awkward cases testable at all — a table that vanished, a coordinator that is down, a + * partition spec whose map iterates in a different order than the partition columns. The mapping is + * checked against a live cluster in {@link FlussMetadataClusterTest}. */ public class FlussConnectorMetadataTest { + private static final TablePath LOG_TABLE = TablePath.of("db", "log_table"); + private static final TablePath PK_TABLE = TablePath.of("db", "pk_table"); + + private static FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { + return new FlussConnectorMetadata(adminOps, FlussTypeMapping.Options.DEFAULT); + } + + /** A non-partitioned log table: two columns, one of them commented, three buckets. */ + private static RecordingFlussAdminOps withLogTable() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.BIGINT()) + .column("payload", DataTypes.STRING(), "what happened") + .buckets(3) + .comment("the log") + .build()); + return adminOps; + } + + /** A partitioned primary-key table that is tiered into a paimon lake. */ + private static RecordingFlussAdminOps withDataLakePkTable() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("dt", DataTypes.STRING()) + .column("region", DataTypes.STRING()) + .column("id", DataTypes.BIGINT()) + .column("amount", DataTypes.DECIMAL(20, 4)) + .primaryKey("dt", "region", "id") + .partitionedBy("dt", "region") + .buckets(2, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/tmp/lake") + .build()); + return adminOps; + } + @Test public void listingIsPassedThroughToTheClusterUnchanged() { // Fluss already has database/table names in Doris's own shape, so nothing here may invent, @@ -37,7 +103,7 @@ public void listingIsPassedThroughToTheClusterUnchanged() { adminOps.tablesByDatabase.put("fluss_db", Arrays.asList("log_table", "pk_table")); adminOps.tablesByDatabase.put("empty_db", Collections.emptyList()); - FlussConnectorMetadata metadata = new FlussConnectorMetadata(adminOps); + FlussConnectorMetadata metadata = metadata(adminOps); Assertions.assertEquals(Arrays.asList("fluss_db", "MixedCase"), metadata.listDatabaseNames(null)); Assertions.assertEquals(Arrays.asList("log_table", "pk_table"), metadata.listTableNames(null, "fluss_db")); @@ -50,4 +116,309 @@ public void listingIsPassedThroughToTheClusterUnchanged() { "databaseExists(fluss_db)", "databaseExists(absent_db)"), adminOps.calls); } + + @Test + public void theHandleSnapshotsWhatSplitPlanningWillNeed() { + // Planning must not have to re-read the table to learn which scanner family reads it, how many + // buckets to split into, or where the lake is; and reading those fields one by one later could + // straddle an ALTER. Everything below is therefore captured at handle time. + FlussTableHandle handle = (FlussTableHandle) metadata(withDataLakePkTable()) + .getTableHandle(null, "db", "pk_table").orElseThrow(AssertionError::new); + + Assertions.assertEquals("db", handle.getDatabaseName()); + Assertions.assertEquals("pk_table", handle.getTableName()); + Assertions.assertTrue(handle.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "region", "id"), handle.getPrimaryKeys()); + Assertions.assertEquals(Arrays.asList("dt", "region"), handle.getPartitionKeys()); + Assertions.assertTrue(handle.isPartitioned()); + Assertions.assertEquals(Collections.singletonList("id"), handle.getBucketKeys()); + Assertions.assertEquals(2, handle.getBucketCount()); + Assertions.assertTrue(handle.isDataLakeEnabled()); + Assertions.assertEquals("paimon", handle.getDataLakeFormat()); + // The lake catalog's own connection settings ride on the fluss table properties (the fluss + // coordinator merges them in); the Doris catalog is told nothing but bootstrap servers, so this + // map is the only place a union read can learn where the lake lives. + Assertions.assertEquals("filesystem", handle.getProperties().get("table.datalake.paimon.metastore")); + Assertions.assertEquals("/tmp/lake", handle.getProperties().get("table.datalake.paimon.warehouse")); + } + + @Test + public void logTableIsNotMistakenForPkOrLakeTable() { + // The negative side of the handle: these three flags pick the scan strategy, so a default that + // drifted to "true" would silently route a plain log table down the merge-on-read path. + FlussTableHandle handle = (FlussTableHandle) metadata(withLogTable()) + .getTableHandle(null, "db", "log_table").orElseThrow(AssertionError::new); + + Assertions.assertFalse(handle.hasPrimaryKey()); + Assertions.assertFalse(handle.isPartitioned()); + Assertions.assertFalse(handle.isDataLakeEnabled()); + Assertions.assertNull(handle.getDataLakeFormat()); + Assertions.assertEquals(3, handle.getBucketCount()); + } + + @Test + public void missingTableIsEmptyButUnreachableClusterIsAnError() { + RecordingFlussAdminOps adminOps = withLogTable(); + Assertions.assertEquals(Optional.empty(), metadata(adminOps).getTableHandle(null, "db", "absent")); + + // The distinction that matters: reporting "no such table" for a cluster that is merely down + // turns a broken catalog into an empty-looking one — the user never sees the real cause, and a + // CREATE TABLE IF NOT EXISTS would believe the name is free. + adminOps.failure = new IllegalStateException("coordinator unreachable"); + IllegalStateException e = Assertions.assertThrows(IllegalStateException.class, + () -> metadata(adminOps).getTableHandle(null, "db", "log_table")); + Assertions.assertEquals("coordinator unreachable", e.getMessage()); + } + + @Test + public void theSchemaCarriesColumnsCommentsAndTheirTypes() { + FlussConnectorMetadata metadata = metadata(withLogTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + + Assertions.assertEquals("log_table", schema.getTableName()); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + Assertions.assertEquals(Arrays.asList("id", "payload"), names(schema.getColumns())); + Assertions.assertEquals(ConnectorType.of("BIGINT"), schema.getColumns().get(0).getType()); + // Column comments come from the fluss schema, NOT from the table's row type: fluss rebuilds that + // row type from the schema and drops every description on the way, so a mapping that read + // getRowType() would report every column as undocumented. + Assertions.assertEquals("what happened", schema.getColumns().get(1).getComment()); + Assertions.assertEquals("the log", metadata.getTableComment(null, "db", "log_table")); + } + + @Test + public void everyColumnIsReportedNullableEvenWhenFlussSaysNotNull() { + // Fluss marks primary-key columns NOT NULL. Propagating that would let the planner fold + // null-rejecting predicates on this path only, while the same table read through its lake + // sibling (served by the paimon connector, which reports everything nullable) would keep them — + // one table, two plans. Doris external tables report nullable throughout for this reason. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + for (ConnectorColumn column : metadata.getTableSchema(null, handle).getColumns()) { + Assertions.assertTrue(column.isNullable(), column.getName() + " must be reported nullable"); + } + } + + @Test + public void partitionedTableAdvertisesItsPartitionColumns() { + // Without this property fe-core treats the table as unpartitioned: no error, just partition + // pruning quietly gone. The value keeps the partition-column ORDER, which is what the partition + // names below are zipped against. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + Map properties = metadata.getTableSchema(null, handle).getProperties(); + Assertions.assertEquals("dt,region", + properties.get(ConnectorTableSchema.PARTITION_COLUMNS_KEY)); + // The fluss table's own properties stay visible so SHOW CREATE TABLE can render them. + Assertions.assertEquals("true", properties.get("table.datalake.enabled")); + + FlussConnectorMetadata logMetadata = metadata(withLogTable()); + ConnectorTableHandle logHandle = logMetadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertFalse( + logMetadata.getTableSchema(null, logHandle).getProperties() + .containsKey(ConnectorTableSchema.PARTITION_COLUMNS_KEY), + "an unpartitioned table must not claim partition columns"); + } + + @Test + public void timestampWithLocalTimeZoneKeepsItsMarkerUnderEitherMapping() { + // The marker is what DESCRIBE prints in Extra, and it describes the SOURCE column, so it must + // not depend on whether enable.mapping.timestamp_tz mapped the column to TIMESTAMPTZ or to a + // plain DATETIME. Losing it under one mapping would make the same column self-describe + // differently depending on a catalog switch. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("event_time", DataTypes.TIMESTAMP_LTZ(6)) + .column("local_time", DataTypes.TIMESTAMP(6)) + .buckets(1) + .build()); + + for (boolean mapTimestampTz : new boolean[] {false, true}) { + FlussConnectorMetadata metadata = new FlussConnectorMetadata( + adminOps, new FlussTypeMapping.Options(false, mapTimestampTz)); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + List columns = metadata.getTableSchema(null, handle).getColumns(); + Assertions.assertTrue(columns.get(0).isWithTimeZone(), + "TIMESTAMP_LTZ must keep the marker with timestamp_tz mapping = " + mapTimestampTz); + Assertions.assertFalse(columns.get(1).isWithTimeZone(), + "a plain TIMESTAMP must not be marked as zoned"); + } + } + + @Test + public void columnHandlesAreKeyedByNameAndCarryTheFieldIndex() { + // The index is the projection fluss is asked for later; keying by name is how the engine's slots + // find their handle. A mismatch between the two would project the wrong column, not fail. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + Map handles = metadata.getColumnHandles(null, handle); + + Assertions.assertEquals(Arrays.asList("dt", "region", "id", "amount"), + new ArrayList<>(handles.keySet())); + Assertions.assertEquals(new FlussColumnHandle("id", 2), handles.get("id")); + Assertions.assertEquals(3, ((FlussColumnHandle) handles.get("amount")).getFieldIndex()); + } + + @Test + public void partitionsAreRenderedWithDorisNamesInPartitionColumnOrder() { + // Fluss spells a partition "2026_08_02$eu"; Doris spells it "dt=2026_08_02/region=eu", which is + // what fe-core parses back for SHOW PARTITIONS and the partition_values function. The ordered + // values are supplied explicitly because fe-core zips them positionally against the partition + // columns — and it does that silently, so a wrong order mis-assigns values rather than failing. + RecordingFlussAdminOps adminOps = withDataLakePkTable(); + adminOps.partitionsByTable.put(PK_TABLE, Arrays.asList( + partition(1L, "region", "eu", "dt", "2026_08_02"), + partition(2L, "dt", "2026_08_03", "region", "us"))); + + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(2, partitions.size()); + // The first fixture's spec deliberately lists region before dt: the rendering must follow the + // partition COLUMNS, not whatever order the spec happens to iterate in. + Assertions.assertEquals("dt=2026_08_02/region=eu", partitions.get(0).getPartitionName()); + Assertions.assertEquals(Arrays.asList("2026_08_02", "eu"), partitions.get(0).getOrderedPartitionValues()); + Assertions.assertEquals("2026_08_02", partitions.get(0).getPartitionValues().get("dt")); + Assertions.assertEquals("eu", partitions.get(0).getPartitionValues().get("region")); + Assertions.assertEquals("dt=2026_08_03/region=us", partitions.get(1).getPartitionName()); + + Assertions.assertEquals(Arrays.asList("dt=2026_08_02/region=eu", "dt=2026_08_03/region=us"), + metadata.listPartitionNames(null, handle)); + } + + @Test + public void anUnpartitionedTableNeverAsksTheClusterForPartitions() { + // Asking fluss for the partitions of an unpartitioned table is an error there, and the handle + // already knows the answer is "none" — so this must be decided locally, not round-tripped. + RecordingFlussAdminOps adminOps = withLogTable(); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + Assertions.assertEquals(Collections.emptyList(), metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertEquals(Collections.emptyList(), metadata.listPartitionNames(null, handle)); + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "no partition call should have been made, calls were: " + adminOps.calls); + } + + @Test + public void statisticsAreARowCountOrNothingAtAll() { + RecordingFlussAdminOps adminOps = withLogTable(); + adminOps.statsByTable.put(LOG_TABLE, new TableStats(4200L)); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableStatistics stats = metadata.getTableStatistics(null, handle) + .orElseThrow(AssertionError::new); + Assertions.assertEquals(4200L, stats.getRowCount()); + // Fluss reports no data size; -1 (unknown) says so instead of implying zero bytes, which would + // make the optimizer treat the table as free to scan. + Assertions.assertEquals(-1L, stats.getDataSize()); + + // A table whose statistics are off reports zero, which is not the same as "empty" — report + // unknown so the estimate falls back instead of pinning the table at zero rows. + adminOps.statsByTable.put(LOG_TABLE, new TableStats(0L)); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + + // Statistics are best effort: analysis and SHOW must not fail because the coordinator hiccuped. + adminOps.failure = new IllegalStateException("coordinator unreachable"); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + } + + @Test + public void oneStatementReadsTheTableOnceAndTheNextStatementReadsItAgain() { + // Four metadata questions about one table are one coordinator round trip. Beyond the cost, it is + // what keeps a statement self-consistent: a concurrent ALTER between two of these calls would + // otherwise build a plan from two different schema versions. + RecordingFlussAdminOps adminOps = withLogTable(); + FlussConnectorMetadata metadata = metadata(adminOps); + FlussTestSession session = new FlussTestSession(7L, "query-1"); + + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "log_table") + .orElseThrow(AssertionError::new); + metadata.getTableSchema(session, handle); + metadata.getColumnHandles(session, handle); + metadata.getTableComment(session, "db", "log_table"); + + Assertions.assertEquals(1, countCalls(adminOps, "getTableInfo"), + "expected one fetch per statement, calls were: " + adminOps.calls); + + // A second statement must NOT reuse the first one's view — this memo is per statement, never a + // cache, or an external ALTER would stay invisible until the FE restarts. + FlussTestSession next = new FlussTestSession(7L, "query-2"); + metadata.getTableHandle(next, "db", "log_table"); + Assertions.assertEquals(2, countCalls(adminOps, "getTableInfo")); + + // And with no session at all (offline planning) nothing is shared: load every time. + metadata.getTableHandle(null, "db", "log_table"); + metadata.getTableHandle(null, "db", "log_table"); + Assertions.assertEquals(4, countCalls(adminOps, "getTableInfo")); + } + + @Test + public void theTableDescriptorIsTheGenericFileScanShape() { + // A fluss scan reaches the BE through the same file scan node the lake connectors use, so the + // descriptor is the hive-shaped one they send. It has to name the right table: the BE looks the + // descriptor up by id and reports the names in profiles. + TTableDescriptor descriptor = metadata(withLogTable()) + .buildTableDescriptor(null, 42L, "log_table", "db", "log_table", 2, 9L); + + Assertions.assertEquals(TTableType.HIVE_TABLE, descriptor.getTableType()); + Assertions.assertEquals(42L, descriptor.getId()); + Assertions.assertEquals(2, descriptor.getNumCols()); + Assertions.assertEquals("log_table", descriptor.getTableName()); + Assertions.assertEquals("db", descriptor.getDbName()); + Assertions.assertEquals("db", descriptor.getHiveTable().getDbName()); + Assertions.assertEquals("log_table", descriptor.getHiveTable().getTableName()); + } + + @Test + public void tableThatVanishesIsReportedMissingNotBroken() { + // getTableHandle discriminates on fluss's own not-exists exception, so it must keep working when + // that exception arrives from a memoized loader rather than straight from the admin call. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + FlussTestSession session = new FlussTestSession(1L, "query-1"); + Assertions.assertEquals(Optional.empty(), + metadata(adminOps).getTableHandle(session, "db", "gone")); + Assertions.assertThrows(TableNotExistException.class, + () -> adminOps.getTableInfo(TablePath.of("db", "gone"))); + } + + private static long countCalls(RecordingFlussAdminOps adminOps, String method) { + return adminOps.calls.stream().filter(call -> call.startsWith(method + "(")).count(); + } + + private static List names(List columns) { + List names = new ArrayList<>(columns.size()); + for (ConnectorColumn column : columns) { + names.add(column.getName()); + } + return names; + } + + /** A fluss partition whose spec is given as key/value pairs, in the caller's order. */ + private static PartitionInfo partition(long partitionId, String... keyValues) { + Map spec = new LinkedHashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + spec.put(keyValues[i], keyValues[i + 1]); + } + return new PartitionInfo(partitionId, + ResolvedPartitionSpec.fromPartitionSpec( + new ArrayList<>(spec.keySet()), new PartitionSpec(spec)), + null); + } } diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java new file mode 100644 index 00000000000000..d13a4c499d9cbf --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTestResult; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The metadata path against a real fluss cluster started in this JVM. + * + *

{@link FlussConnectorMetadataTest} pins the mapping against canned {@code TableInfo}s, which + * proves the logic but not the premise: it cannot tell whether a real cluster actually reports column + * comments where we read them, orders schema columns the way we assume, or answers the calls we make + * at all. This starts a coordinator, a tablet server and an embedded ZooKeeper, creates tables through + * the fluss Java client, and reads them back through the connector — every hop real except Doris's own + * plugin classloading. + * + *

It is deliberately named {@code ...Test} rather than {@code ...ITCase}: surefire's default + * includes do not match {@code *ITCase}, so that name would leave the whole class silently unexecuted + * with a green build. + */ +public class FlussMetadataClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** + * Each test gets its own database. The cluster extension drops every non-built-in database after + * each test, so a shared fixture would vanish after the first one; a fresh NAME each time (rather + * than re-creating the same one) also keeps the fluss client's metadata cache from ever holding a + * dropped table's id for a path that came back. + */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() throws Exception { + Configuration clientConf = FLUSS_CLUSTER.getClientConfig(); + connection = ConnectionFactory.createConnection(clientConf); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_metadata_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "log_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .withComment("the row id") + .column("name", DataTypes.STRING()) + .column("price", DataTypes.DECIMAL(20, 4)) + .column("event_time", DataTypes.TIMESTAMP_LTZ(6)) + .column("started", DataTypes.TIME(3)) + .column("tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("nested", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT(), "inner doc"))) + .build()) + .distributedBy(3) + .comment("a log table") + .build(), + true).get(); + + admin.createTable(TablePath.of(db, "pk_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("dt", DataTypes.STRING()) + .column("id", DataTypes.BIGINT()) + .column("amount", DataTypes.DOUBLE()) + .primaryKey("dt", "id") + .build()) + .partitionedBy("dt") + .distributedBy(2, "id") + .build(), + true).get(); + admin.createPartition(TablePath.of(db, "pk_table"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(TablePath.of(db, "pk_table"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + private static ConnectorMetadata metadata() { + return connector.getMetadata(new FlussTestSession(1L, "cluster-query")); + } + + @Test + public void theCatalogSeesTheClusterAndItsNamespaces() { + ConnectorTestResult result = connector.testConnection(null); + Assertions.assertTrue(result.isSuccess(), () -> "connectivity test failed: " + result); + + ConnectorMetadata metadata = metadata(); + Assertions.assertTrue(metadata.listDatabaseNames(null).contains(db)); + Assertions.assertTrue(metadata.databaseExists(null, db)); + Assertions.assertFalse(metadata.databaseExists(null, "no_such_db")); + List tables = metadata.listTableNames(null, db); + Assertions.assertTrue(tables.containsAll(Arrays.asList("log_table", "pk_table")), + "expected both tables, got " + tables); + Assertions.assertEquals(Optional.empty(), metadata.getTableHandle(null, db, "no_such_table")); + } + + @Test + public void schemaComesBackFromTheClusterWithTypesAndComments() { + // The premise the unit tests cannot check: that a real cluster reports these facts where this + // connector reads them. Column comments are the sharpest case — fluss keeps them on the schema + // and drops them from the table's row type, so reading the wrong one loses every comment. + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + Map columns = byName(schema.getColumns()); + + Assertions.assertEquals(ConnectorType.of("BIGINT"), columns.get("id").getType()); + Assertions.assertEquals("the row id", columns.get("id").getComment()); + Assertions.assertEquals(ConnectorType.of("STRING"), columns.get("name").getType()); + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 20, 4), columns.get("price").getType()); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), columns.get("event_time").getType()); + Assertions.assertTrue(columns.get("event_time").isWithTimeZone()); + // TIME survives the round trip as an unsupported column rather than making the table unloadable. + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), columns.get("started").getType()); + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("STRING"), ConnectorType.of("INT")), + columns.get("tags").getType()); + ConnectorType nested = columns.get("nested").getType(); + Assertions.assertEquals("STRUCT", nested.getTypeName()); + Assertions.assertEquals(Collections.singletonList("a"), nested.getFieldNames()); + Assertions.assertEquals("inner doc", nested.getChildComment(0)); + + Assertions.assertEquals("a log table", metadata.getTableComment(null, db, "log_table")); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + + // Column handles are positional, and the position must be the cluster's column order. + Map handles = metadata.getColumnHandles(null, handle); + Assertions.assertEquals(new FlussColumnHandle("id", 0), handles.get("id")); + Assertions.assertEquals(new FlussColumnHandle("nested", 6), handles.get("nested")); + } + + @Test + public void theHandleReflectsHowTheClusterCreatedTheTable() { + ConnectorMetadata metadata = metadata(); + FlussTableHandle logHandle = (FlussTableHandle) metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertFalse(logHandle.hasPrimaryKey()); + Assertions.assertFalse(logHandle.isPartitioned()); + Assertions.assertFalse(logHandle.isDataLakeEnabled()); + Assertions.assertEquals(3, logHandle.getBucketCount()); + // A real cluster assigns the ids; all the fixtures can say is that they were carried through. + Assertions.assertTrue(logHandle.getTableId() > 0, "expected a cluster-assigned table id"); + + FlussTableHandle pkHandle = (FlussTableHandle) metadata.getTableHandle(null, db, "pk_table") + .orElseThrow(AssertionError::new); + Assertions.assertTrue(pkHandle.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "id"), pkHandle.getPrimaryKeys()); + Assertions.assertEquals(Collections.singletonList("dt"), pkHandle.getPartitionKeys()); + Assertions.assertEquals(Collections.singletonList("id"), pkHandle.getBucketKeys()); + Assertions.assertEquals(2, pkHandle.getBucketCount()); + } + + @Test + public void partitionsAreListedAsTheClusterCreatedThem() { + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "pk_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + List names = new ArrayList<>(); + for (ConnectorPartitionInfo partition : partitions) { + names.add(partition.getPartitionName()); + Assertions.assertEquals(1, partition.getOrderedPartitionValues().size()); + } + Collections.sort(names); + Assertions.assertEquals(Arrays.asList("dt=2026_08_02", "dt=2026_08_03"), names); + + Assertions.assertEquals("dt", + metadata.getTableSchema(null, handle).getProperties() + .get(ConnectorTableSchema.PARTITION_COLUMNS_KEY)); + + // The unpartitioned table must come back empty, not fail: fluss rejects the partition call for + // such a table, so this also proves the connector never makes it. + ConnectorTableHandle logHandle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertEquals(Collections.emptyList(), + metadata.listPartitions(null, logHandle, Optional.empty())); + } + + @Test + public void statisticsOfAFreshTableAreUnknownRatherThanZero() { + // Nothing has been written and statistics are off by default, so fluss reports no rows. Reporting + // that as a row count of zero would tell the optimizer the table is empty. + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + } + + private static Map byName(List columns) { + Map byName = new LinkedHashMap<>(); + for (ConnectorColumn column : columns) { + byName.put(column.getName(), column); + } + return byName; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java new file mode 100644 index 00000000000000..79f5b41562726e --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Guards the statement-scope naming norm: every namespace this connector declares is prefixed with its + * own connector type. The scope is keyed by {@code (catalogId, db, table, queryId)} and shared across + * connectors, so an unprefixed namespace could hand a gateway statement another connector's value for + * the same table coordinate — which surfaces as a ClassCastException, not as a wrong answer, but only + * on the gateway path nobody runs by default. + */ +public class FlussStatementScopeTest { + + @Test + public void allNamespacesArePrefixedWithConnectorType() throws Exception { + // Reflective on purpose: a namespace added later is covered without anyone remembering to come + // back here, and a drift in getType() turns this red on its own. + String prefix = new FlussConnectorProvider().getType() + "."; + int checked = 0; + for (Field field : FlussStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) && field.getType() == String.class + && field.getName().endsWith("_NAMESPACE")) { + field.setAccessible(true); + String namespace = (String) field.get(null); + Assertions.assertTrue(namespace.startsWith(prefix), + field.getName() + " (\"" + namespace + "\") must start with \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java new file mode 100644 index 00000000000000..0b7be5b4d99677 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The handle is the connector's contract with the engine, and the SPI declares it {@link + * java.io.Serializable}. These pin the two ways that contract can break silently. + */ +public class FlussTableHandleTest { + + private static FlussTableHandle handle(String tableName, long tableId, int schemaId) { + Map properties = new LinkedHashMap<>(); + properties.put("table.datalake.enabled", "true"); + properties.put("table.datalake.paimon.warehouse", "/tmp/lake"); + return new FlussTableHandle("db", tableName, tableId, schemaId, true, + Arrays.asList("dt", "id"), Collections.singletonList("id"), 4, + Collections.singletonList("dt"), true, "paimon", properties); + } + + @Test + public void survivesJavaSerializationWithEveryFieldIntact() { + // A field that fails to serialize does not fail loudly here — it comes back null and the plan + // built from it is silently wrong (no lake configuration, zero buckets), so every field the + // scan side reads is checked on the far side of the round trip. + FlussTableHandle restored = roundTrip(handle("pk_table", 7L, 3)); + + Assertions.assertEquals("db", restored.getDatabaseName()); + Assertions.assertEquals("pk_table", restored.getTableName()); + Assertions.assertEquals(7L, restored.getTableId()); + Assertions.assertEquals(3, restored.getSchemaId()); + Assertions.assertTrue(restored.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "id"), restored.getPrimaryKeys()); + Assertions.assertEquals(Collections.singletonList("id"), restored.getBucketKeys()); + Assertions.assertEquals(4, restored.getBucketCount()); + Assertions.assertEquals(Collections.singletonList("dt"), restored.getPartitionKeys()); + Assertions.assertTrue(restored.isPartitioned()); + Assertions.assertTrue(restored.isDataLakeEnabled()); + Assertions.assertEquals("paimon", restored.getDataLakeFormat()); + Assertions.assertEquals("/tmp/lake", restored.getProperties().get("table.datalake.paimon.warehouse")); + Assertions.assertEquals(handle("pk_table", 7L, 3), restored); + } + + @Test + public void identityIsTheTableAtItsSchemaVersion() { + Assertions.assertEquals(handle("t", 1L, 1), handle("t", 1L, 1)); + Assertions.assertEquals(handle("t", 1L, 1).hashCode(), handle("t", 1L, 1).hashCode()); + + // A different schema version describes a different set of columns; treating the two as the same + // handle would let a cached value from before an ALTER answer for the table after it. + Assertions.assertNotEquals(handle("t", 1L, 1), handle("t", 1L, 2)); + // A table dropped and recreated under the same name is a different table. + Assertions.assertNotEquals(handle("t", 1L, 1), handle("t", 2L, 1)); + Assertions.assertNotEquals(handle("t", 1L, 1), handle("other", 1L, 1)); + } + + private static FlussTableHandle roundTrip(FlussTableHandle handle) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(handle); + } + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (FlussTableHandle) in.readObject(); + } + } catch (Exception e) { + throw new AssertionError("the handle must be serializable", e); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java new file mode 100644 index 00000000000000..7a36ecddbcd06d --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * A session with a real per-statement scope, for tests that assert what a statement shares. + * + *

The scope is a plain map keyed exactly as the engine's is, so the sharing (and the isolation + * between two query ids) is the connector's own, not something this class arranges. + */ +final class FlussTestSession implements ConnectorSession { + + private final long catalogId; + private final String queryId; + private final Map values = new ConcurrentHashMap<>(); + private final ConnectorStatementScope scope = new ConnectorStatementScope() { + @SuppressWarnings("unchecked") + @Override + public T computeIfAbsent(String key, Supplier loader) { + return (T) values.computeIfAbsent(key, ignored -> loader.get()); + } + }; + + FlussTestSession(long catalogId, String queryId) { + this.catalogId = catalogId; + this.queryId = queryId; + } + + @Override + public long getCatalogId() { + return catalogId; + } + + @Override + public String getQueryId() { + return queryId; + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public String getUser() { + return "test_user"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java new file mode 100644 index 00000000000000..2a930096d58742 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Builds the {@link TableInfo} a fluss cluster would hand back, for tests that do not start one. + * + *

It goes through fluss's own {@link TableDescriptor} and {@link TableInfo#of} rather than + * assembling a {@link TableInfo} field by field, so a fixture cannot describe a table fluss would + * refuse to create (a partition key outside the primary key, a missing bucket count) and quietly make + * the connector look correct on input it will never see. + */ +final class FlussTestTables { + + /** Ids a fixture does not care about; a real cluster assigns its own. */ + private static final long TABLE_ID = 1L; + private static final int SCHEMA_ID = 1; + + private FlussTestTables() { + } + + static Builder builder(TablePath tablePath) { + return new Builder(tablePath); + } + + static final class Builder { + + private final TablePath tablePath; + private final Schema.Builder schema = Schema.newBuilder(); + private final Map properties = new LinkedHashMap<>(); + private final List partitionKeys = new ArrayList<>(); + private List bucketKeys = new ArrayList<>(); + private int bucketCount = 1; + private String comment; + + private Builder(TablePath tablePath) { + this.tablePath = tablePath; + } + + Builder column(String name, DataType type) { + schema.column(name, type); + return this; + } + + Builder column(String name, DataType type, String columnComment) { + schema.column(name, type).withComment(columnComment); + return this; + } + + Builder primaryKey(String... columns) { + schema.primaryKey(columns); + return this; + } + + Builder partitionedBy(String... columns) { + partitionKeys.addAll(Arrays.asList(columns)); + return this; + } + + Builder buckets(int count, String... keys) { + this.bucketCount = count; + this.bucketKeys = new ArrayList<>(Arrays.asList(keys)); + return this; + } + + Builder property(String key, String value) { + properties.put(key, value); + return this; + } + + Builder comment(String tableComment) { + this.comment = tableComment; + return this; + } + + TableInfo build() { + TableDescriptor descriptor = TableDescriptor.builder() + .schema(schema.build()) + .partitionedBy(partitionKeys) + .distributedBy(bucketCount, bucketKeys) + .properties(properties) + .comment(comment) + .build(); + return TableInfo.of(tablePath, TABLE_ID, SCHEMA_ID, descriptor, null, 0L, 0L); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java index 90468bf30c6c06..fb47ab4ccf4f4c 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -20,6 +20,7 @@ import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.PartitionSpec; import org.apache.fluss.metadata.TableInfo; @@ -52,6 +53,11 @@ class RecordingFlussAdminOps implements FlussAdminOps { List databases = Collections.emptyList(); final Map> tablesByDatabase = new HashMap<>(); + final Map tableInfos = new HashMap<>(); + final Map> partitionsByTable = new HashMap<>(); + final Map statsByTable = new HashMap<>(); + /** When set, every call throws this instead of answering — the "cluster is unreachable" case. */ + RuntimeException failure; @Override public List listDatabases() { @@ -82,12 +88,26 @@ public boolean tableExists(TablePath tablePath) { @Override public TableInfo getTableInfo(TablePath tablePath) { - throw notProgrammed("getTableInfo"); + calls.add("getTableInfo(" + tablePath + ")"); + if (failure != null) { + throw failure; + } + TableInfo tableInfo = tableInfos.get(tablePath); + if (tableInfo == null) { + // What a real fluss cluster answers for an unknown table; the connector discriminates on it. + throw new TableNotExistException("Table '" + tablePath + "' does not exist."); + } + return tableInfo; } @Override public List listPartitionInfos(TablePath tablePath) { - throw notProgrammed("listPartitionInfos"); + calls.add("listPartitionInfos(" + tablePath + ")"); + List partitions = partitionsByTable.get(tablePath); + if (partitions == null) { + throw new IllegalStateException("no partitions programmed for table '" + tablePath + "'"); + } + return partitions; } @Override @@ -97,7 +117,15 @@ public List listPartitionInfos(TablePath tablePath, PartitionSpec @Override public TableStats getTableStats(TablePath tablePath) { - throw notProgrammed("getTableStats"); + calls.add("getTableStats(" + tablePath + ")"); + if (failure != null) { + throw failure; + } + TableStats stats = statsByTable.get(tablePath); + if (stats == null) { + throw new IllegalStateException("no stats programmed for table '" + tablePath + "'"); + } + return stats; } @Override From da79d189c200b6c69c010ab8c774e6140d3aedfe Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 20:24:16 +0800 Subject: [PATCH 04/35] [feat](fluss) Add the fluss e2e docker environment The regression suites need a fluss cluster to read from. This brings up ZooKeeper, a fluss coordinator and tablet server, and a Flink cluster whose SQL client creates the fixture tables once and then idles, so that `compose up --wait` gates on the fixtures being complete rather than on the containers merely running. Fluss 1.0 is not released, so neither image can be pulled: build-images.sh builds both from a local fluss checkout, which is also why the component is not part of the default component set yet. Two things the servers need because Doris runs outside the compose network: the advertised listeners are the host address plus the published port, and remote.data.dir is bind mounted at the same absolute path on both sides, since BE will read the kv snapshots written there directly. Also adds fluss to both connector lists in build.sh. They were missing it, so the plugin was never packaged or deployed. Co-Authored-By: Claude Opus 5 (1M context) --- build.sh | 4 +- .../docker-compose/fluss/README.md | 88 ++++++ .../docker-compose/fluss/build-images.sh | 111 ++++++++ .../docker-compose/fluss/flink/Dockerfile | 24 ++ .../docker-compose/fluss/fluss.env.tpl | 41 +++ .../docker-compose/fluss/fluss.yaml.tpl | 188 ++++++++++++ .../fluss/scripts/run-init-sql.sh | 91 ++++++ .../docker-compose/fluss/sql/init.sql | 268 ++++++++++++++++++ .../thirdparties/run-thirdparties-docker.sh | 44 ++- regression-test/conf/regression-conf.groovy | 6 + .../fluss/test_fluss_catalog.groovy | 129 +++++++++ 11 files changed, 990 insertions(+), 4 deletions(-) create mode 100644 docker/thirdparties/docker-compose/fluss/README.md create mode 100755 docker/thirdparties/docker-compose/fluss/build-images.sh create mode 100644 docker/thirdparties/docker-compose/fluss/flink/Dockerfile create mode 100644 docker/thirdparties/docker-compose/fluss/fluss.env.tpl create mode 100644 docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl create mode 100755 docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh create mode 100644 docker/thirdparties/docker-compose/fluss/sql/init.sql create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy diff --git a/build.sh b/build.sh index 50539178292547..39e27e91c8badc 100755 --- a/build.sh +++ b/build.sh @@ -726,7 +726,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Connector API, SPI, and plugin modules (loaded at runtime as plugins) modules+=("fe-connector/fe-connector-api") modules+=("fe-connector/fe-connector-spi") - for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg fluss; do if [[ -d "${DORIS_HOME}/fe/fe-connector/fe-connector-${_conn_mod}" ]]; then modules+=("fe-connector/fe-connector-${_conn_mod}") fi @@ -1070,7 +1070,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy connector provider plugins as independent plugin directories. # Each sub-directory is one connector backend loaded at runtime by ConnectorPluginManager. CONN_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/connector" - for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg fluss; do conn_plugin_target="${CONN_PLUGIN_DIR}/${conn_module}" conn_module_dir="${DORIS_HOME}/fe/fe-connector/fe-connector-${conn_module}" if [ ! -d "${conn_module_dir}" ]; then diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md new file mode 100644 index 00000000000000..0688660bee8ace --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -0,0 +1,88 @@ + + +# Fluss regression environment + +Stack: ZooKeeper, a fluss coordinator server, one fluss tablet server, and a +Flink cluster (jobmanager, taskmanager, sql-client). The sql-client container +runs `sql/init.sql` once and then idles; its healthcheck only turns green after +every statement succeeded, so `--wait` gates on the fixtures being complete. + +## Prerequisite: a built fluss checkout + +Fluss 1.0 is not released yet, so neither image can be pulled. `build-images.sh` +builds both from a local source tree, which must be packaged first: + +```bash +git clone https://github.com/apache/fluss.git +mvn -f fluss/pom.xml -pl fluss-dist,fluss-flink/fluss-flink-1.20 -am package -DskipTests +``` + +When fluss 1.0 ships, this step and `build-images.sh` are replaced by the +official `apache/fluss` and Flink images. + +## Start / stop + +```bash +CONTAINER_UID=doris-e2e-- FLUSS_SOURCE_DIR=/path/to/fluss \ + bash docker/thirdparties/run-thirdparties-docker.sh -c fluss + +CONTAINER_UID=doris-e2e-- bash docker/thirdparties/run-thirdparties-docker.sh -c fluss --stop +``` + +`build-images.sh` reuses images that already exist; set +`FLUSS_DOCKER_REUSE_IMAGES=0` to force a rebuild after changing the fluss +checkout. + +Then enable the suites in `regression-test/conf/regression-conf.groovy`: + +```groovy +enableFlussTest=true +``` + +## Ports and paths + +| What | Host port | +|---|---| +| ZooKeeper | 22181 | +| fluss coordinator server | 19123 | +| fluss tablet server | 19124 | +| Flink jobmanager UI | 18085 | + +The servers advertise `:`, because Doris FE/BE run on +the host rather than inside the compose network. + +`remote.data.dir` is bind mounted at the same absolute path inside the +containers and on the host (`data/remote`): Doris BE reads the kv snapshots and +remote log segments written there directly, so the two sides must agree on the +path string. + +## Fixtures + +`sql/init.sql` recreates database `fluss_test` from scratch on every start: + +| Table | Shape | +|---|---| +| `log_basic` | log table, 3 rows, table and column comments | +| `log_types` | log table, one column per mapped fluss type, plus an all-NULL row | +| `log_part` | log table partitioned by `dt`, partitions `20260101`, `20260102`, `20260103` | +| `pk_basic` | primary-key table, one updated row and one deleted row | +| `pk_types` | primary-key table with the same type coverage as `log_types` | + +Data-lake tables and the tiering service are added when union read lands. diff --git a/docker/thirdparties/docker-compose/fluss/build-images.sh b/docker/thirdparties/docker-compose/fluss/build-images.sh new file mode 100755 index 00000000000000..6676248bd6fe13 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/build-images.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################ +# Builds the two images the fluss regression stack runs on. +# +# Fluss 1.0 is not released, so neither image can be pulled: both are built +# from a local fluss source checkout that has already been packaged. Once +# fluss 1.0 ships, this script is replaced by pulling the official images. +# +# Required: +# FLUSS_SOURCE_DIR path to a built fluss checkout +# Optional: +# FLUSS_VERSION fluss version in that checkout (default 1.0-SNAPSHOT) +# FLINK_BASE_IMAGE base Flink image (default flink:1.20.0-scala_2.12-java17) +# FLUSS_FLINK_CONNECTOR_MODULE fluss connector module matching the base image +# FLUSS_DOCKER_REUSE_IMAGES 1 = skip the build when both tags already exist +################################################################ + +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +# Image tags live in fluss.env.tpl so the compose file and this script cannot +# drift apart. The template's other entries reference variables that are empty +# here; only the two image tags are read. +# shellcheck source=/dev/null +. "${SCRIPT_DIR}/fluss.env.tpl" + +FLUSS_VERSION="${FLUSS_VERSION:-1.0-SNAPSHOT}" +FLINK_BASE_IMAGE="${FLINK_BASE_IMAGE:-flink:1.20.0-scala_2.12-java17}" +FLUSS_FLINK_CONNECTOR_MODULE="${FLUSS_FLINK_CONNECTOR_MODULE:-fluss-flink-1.20}" + +if [[ -z "${DOCKER_USE_SUDO+x}" ]]; then + if [[ "$(uname -s)" == "Darwin" ]]; then + DOCKER_USE_SUDO=0 + else + DOCKER_USE_SUDO=1 + fi +fi + +docker_cli() { + if [[ "${DOCKER_USE_SUDO}" -eq 1 ]]; then + sudo docker "$@" + else + docker "$@" + fi +} + +image_exists() { + docker_cli image inspect "$1" >/dev/null 2>&1 +} + +if [[ "${FLUSS_DOCKER_REUSE_IMAGES}" == "1" ]] && + image_exists "${FLUSS_SERVER_IMAGE}" && image_exists "${FLUSS_FLINK_IMAGE}"; then + echo "Reusing existing images ${FLUSS_SERVER_IMAGE} and ${FLUSS_FLINK_IMAGE}" + exit 0 +fi + +if [[ -z "${FLUSS_SOURCE_DIR}" ]]; then + echo "ERROR: FLUSS_SOURCE_DIR is not set." >&2 + echo " Clone https://github.com/apache/fluss, build it, and point" >&2 + echo " FLUSS_SOURCE_DIR at the checkout." >&2 + exit 1 +fi + +DIST_DIR="${FLUSS_SOURCE_DIR}/fluss-dist/target/fluss-${FLUSS_VERSION}-bin/fluss-${FLUSS_VERSION}" +CONNECTOR_JAR="${FLUSS_SOURCE_DIR}/fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE}/target/${FLUSS_FLINK_CONNECTOR_MODULE}-${FLUSS_VERSION}.jar" + +if [[ ! -d "${DIST_DIR}" || ! -f "${CONNECTOR_JAR}" ]]; then + echo "ERROR: fluss build output is missing:" >&2 + [[ -d "${DIST_DIR}" ]] || echo " ${DIST_DIR}" >&2 + [[ -f "${CONNECTOR_JAR}" ]] || echo " ${CONNECTOR_JAR}" >&2 + echo " Build them with:" >&2 + echo " mvn -f ${FLUSS_SOURCE_DIR}/pom.xml -pl fluss-dist,fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE} -am package -DskipTests" >&2 + exit 1 +fi + +BUILD_CONTEXT="$(mktemp -d)" +trap 'rm -rf "${BUILD_CONTEXT}"' EXIT + +echo "Building ${FLUSS_SERVER_IMAGE} from ${DIST_DIR}" +mkdir -p "${BUILD_CONTEXT}/server" +cp -r "${DIST_DIR}" "${BUILD_CONTEXT}/server/build-target" +cp "${FLUSS_SOURCE_DIR}/docker/fluss/Dockerfile" "${BUILD_CONTEXT}/server/Dockerfile" +cp "${FLUSS_SOURCE_DIR}/docker/fluss/docker-entrypoint.sh" "${BUILD_CONTEXT}/server/docker-entrypoint.sh" +docker_cli build -t "${FLUSS_SERVER_IMAGE}" "${BUILD_CONTEXT}/server" + +echo "Building ${FLUSS_FLINK_IMAGE} from ${FLINK_BASE_IMAGE} + $(basename "${CONNECTOR_JAR}")" +mkdir -p "${BUILD_CONTEXT}/flink/lib" +cp "${CONNECTOR_JAR}" "${BUILD_CONTEXT}/flink/lib/" +cp "${SCRIPT_DIR}/flink/Dockerfile" "${BUILD_CONTEXT}/flink/Dockerfile" +docker_cli build --build-arg "FLINK_BASE_IMAGE=${FLINK_BASE_IMAGE}" \ + -t "${FLUSS_FLINK_IMAGE}" "${BUILD_CONTEXT}/flink" + +echo "Built ${FLUSS_SERVER_IMAGE} and ${FLUSS_FLINK_IMAGE}" diff --git a/docker/thirdparties/docker-compose/fluss/flink/Dockerfile b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile new file mode 100644 index 00000000000000..369eb822fdb42d --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Flink image used to prepare the fluss regression data: stock Flink plus the +# fluss connector. build-images.sh assembles lib/ from a local fluss checkout, +# because fluss 1.0 is not released yet. +ARG FLINK_BASE_IMAGE=flink:1.20.0-scala_2.12-java17 +FROM ${FLINK_BASE_IMAGE} + +COPY lib/ /opt/flink/lib/ diff --git a/docker/thirdparties/docker-compose/fluss/fluss.env.tpl b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl new file mode 100644 index 00000000000000..44b6acaf2b9463 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Rendered to fluss.env by run-thirdparties-docker.sh (envsubst). +# build-images.sh also sources this template directly for the image tags, so +# keep the two FLUSS_*_IMAGE lines free of variable references. + +DOCKER_FLUSS_ZOOKEEPER_EXTERNAL_PORT=22181 +DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT=19123 +DOCKER_FLUSS_TABLET_EXTERNAL_PORT=19124 +DOCKER_FLUSS_FLINK_JOBMANAGER_EXTERNAL_PORT=18085 + +# Fluss 1.0 is not released yet, so there is no published image to pull. +# build-images.sh builds both tags from a local fluss source checkout. +FLUSS_SERVER_IMAGE=doris-fluss-server:1.0-SNAPSHOT-local +FLUSS_FLINK_IMAGE=doris-fluss-flink:1.20.0-fluss-1.0-SNAPSHOT-local + +# Address the fluss servers advertise to clients. Doris FE/BE run on the host, +# so the servers must hand out the host address plus the published ports, not +# their in-container hostnames. +FLUSS_HOST_IP=${IP_HOST} + +# remote.data.dir holds remote log segments and kv snapshots. Doris BE reads +# those files directly (primary-key table reads), so the directory is bind +# mounted at the SAME absolute path inside the containers and on the host. +FLUSS_REMOTE_DATA_DIR=${FLUSS_COMPOSE_DIR}/data/remote diff --git a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl new file mode 100644 index 00000000000000..f9b419e69527b8 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl @@ -0,0 +1,188 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +networks: + doris--fluss--network: + ipam: + driver: default + config: + - subnet: 168.52.0.0/24 + +services: + doris--fluss-zookeeper: + image: zookeeper:3.9.2 + container_name: doris--fluss-zookeeper + hostname: doris--fluss-zookeeper + restart: always + ports: + - ${DOCKER_FLUSS_ZOOKEEPER_EXTERNAL_PORT}:2181 + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/2181' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-coordinator: + image: ${FLUSS_SERVER_IMAGE} + container_name: doris--fluss-coordinator + hostname: doris--fluss-coordinator + command: coordinatorServer + depends_on: + doris--fluss-zookeeper: + condition: service_healthy + ports: + - ${DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT}:9123 + environment: + # Doris FE/BE run on the host, so the servers must advertise the host + # address plus the published port: everything (host clients, the flink + # containers and the servers among themselves) then talks over it. + - | + FLUSS_PROPERTIES= + zookeeper.address: doris--fluss-zookeeper:2181 + bind.listeners: FLUSS://0.0.0.0:9123 + advertised.listeners: FLUSS://${FLUSS_HOST_IP}:${DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT} + remote.data.dir: ${FLUSS_REMOTE_DATA_DIR} + default.bucket.number: 3 + default.replication.factor: 1 + volumes: + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-tablet-server: + image: ${FLUSS_SERVER_IMAGE} + container_name: doris--fluss-tablet-server + hostname: doris--fluss-tablet-server + command: tabletServer + depends_on: + doris--fluss-coordinator: + condition: service_healthy + ports: + - ${DOCKER_FLUSS_TABLET_EXTERNAL_PORT}:9123 + environment: + - | + FLUSS_PROPERTIES= + zookeeper.address: doris--fluss-zookeeper:2181 + bind.listeners: FLUSS://0.0.0.0:9123 + advertised.listeners: FLUSS://${FLUSS_HOST_IP}:${DOCKER_FLUSS_TABLET_EXTERNAL_PORT} + tablet-server.id: 0 + data.dir: /tmp/fluss/data + remote.data.dir: ${FLUSS_REMOTE_DATA_DIR} + default.bucket.number: 3 + default.replication.factor: 1 + # The fixtures write kilobytes, but the guard measures the whole host + # disk: on a build machine that happens to sit above the threshold it + # rejects every write, and the client retries ~2^31 times instead of + # failing, so the environment hangs instead of reporting anything. + server.data-disk.write-limit-ratio: 1.0 + volumes: + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-jobmanager: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-jobmanager + hostname: doris--fluss-jobmanager + command: jobmanager + ports: + - ${DOCKER_FLUSS_FLINK_JOBMANAGER_EXTERNAL_PORT}:8081 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + rest.address: doris--fluss-jobmanager + rest.bind-address: 0.0.0.0 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8081/overview >/dev/null"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-taskmanager: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-taskmanager + hostname: doris--fluss-taskmanager + command: taskmanager + depends_on: + doris--fluss-jobmanager: + condition: service_healthy + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + taskmanager.numberOfTaskSlots: 4 + taskmanager.memory.process.size: 2048m + taskmanager.memory.task.off-heap.size: 128m + # The taskmanager RPC port is ephemeral, so health means "registered with + # the jobmanager": that is also exactly what submitting a job needs. + healthcheck: + test: ["CMD-SHELL", "curl -sf http://doris--fluss-jobmanager:8081/taskmanagers | grep -q '\"id\"'"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + # One-shot data preparation: runs sql/init.sql through the Flink SQL client, + # then keeps running so that `compose up --wait` has a healthy service to + # gate on. The marker file only appears when every statement succeeded. + doris--fluss-sql-client: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-sql-client + hostname: doris--fluss-sql-client + depends_on: + doris--fluss-tablet-server: + condition: service_healthy + doris--fluss-taskmanager: + condition: service_healthy + # Runs as a command, not as an entrypoint override: the image entrypoint is + # what turns FLINK_PROPERTIES into the config the SQL client submits with. + command: ["/opt/fluss-scripts/run-init-sql.sh"] + environment: + - FLUSS_BOOTSTRAP_SERVERS=doris--fluss-coordinator:9123 + - FLUSS_JOBMANAGER_HOST=doris--fluss-jobmanager + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + rest.address: doris--fluss-jobmanager + volumes: + - ./sql:/opt/fluss-sql:ro + - ./scripts:/opt/fluss-scripts:ro + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/fluss-init/SUCCESS"] + interval: 5s + timeout: 10s + retries: 120 + networks: + - doris--fluss--network diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh new file mode 100755 index 00000000000000..0709b870e6e864 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################ +# Entrypoint of the fluss sql-client container: creates the regression +# fixtures through the Flink SQL client, then idles so that the compose +# healthcheck has something to gate on. The SUCCESS marker is written only +# after every statement succeeded. +################################################################ + +set -eo pipefail + +MARKER_DIR=/tmp/fluss-init +SQL_TEMPLATE=/opt/fluss-sql/init.sql +JOBMANAGER_PORT=8081 +WAIT_SECONDS=180 +SQL_TIMEOUT_SECONDS=900 +ATTEMPTS=3 + +rm -rf "${MARKER_DIR}" +mkdir -p "${MARKER_DIR}" + +wait_for_jobmanager() { + local waited=0 + while ! (exec 3<>"/dev/tcp/${FLUSS_JOBMANAGER_HOST}/${JOBMANAGER_PORT}") >/dev/null 2>&1; do + if ((waited >= WAIT_SECONDS)); then + echo "ERROR: jobmanager ${FLUSS_JOBMANAGER_HOST}:${JOBMANAGER_PORT} not reachable after ${WAIT_SECONDS}s" >&2 + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done +} + +wait_for_jobmanager + +# The bootstrap address is only known at compose time, and the Flink SQL client +# does not expand environment variables inside SQL files. +sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ + "${SQL_TEMPLATE}" >"${MARKER_DIR}/init.sql" + +run_attempt() { + local log="$1" + local status=0 + + # Timeout, because a write that the servers keep rejecting is retried by the + # fluss client practically forever: without it the container just hangs. + timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh -f "${MARKER_DIR}/init.sql" 2>&1 | tee "${log}" + status="${PIPESTATUS[0]}" + if ((status != 0)); then + return "${status}" + fi + # The SQL client stops at the first failing statement but still exits 0, so + # the only honest completion signal is its own output. + if grep -q '\[ERROR\]' "${log}"; then + return 1 + fi + return 0 +} + +# init.sql drops and recreates its database up front, so a retry always starts +# from the same state. Retries exist because the tablet server may still be +# registering with the coordinator when the ports are already open. +for ((attempt = 1; attempt <= ATTEMPTS; attempt++)); do + echo "Running fluss init SQL (attempt ${attempt}/${ATTEMPTS})" + if run_attempt "${MARKER_DIR}/init-attempt-${attempt}.log"; then + touch "${MARKER_DIR}/SUCCESS" + echo "Fluss init SQL finished" + exec tail -f /dev/null + fi + echo "Fluss init SQL failed on attempt ${attempt}" >&2 + sleep 10 +done + +echo "ERROR: fluss init SQL failed after ${ATTEMPTS} attempts" >&2 +exit 1 diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql new file mode 100644 index 00000000000000..71b67269429305 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -0,0 +1,268 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Regression fixtures for the fluss catalog. Data is static: the suites read +-- it, they never write. __FLUSS_BOOTSTRAP_SERVERS__ is substituted by +-- scripts/run-init-sql.sh. +-- +-- Tables backed by the data lake are added together with the tiering service +-- when union read lands; everything here is fluss-only. + +SET 'table.dml-sync' = 'true'; +SET 'parallelism.default' = '2'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = '__FLUSS_BOOTSTRAP_SERVERS__' +); + +USE CATALOG fluss_catalog; + +-- Recreated from scratch so that rerunning this script is idempotent. +DROP DATABASE IF EXISTS fluss_test CASCADE; +CREATE DATABASE fluss_test; +USE fluss_test; + +-- --------------------------------------------------------------------------- +-- log_basic: plain append-only table, carries table and column comments so the +-- catalog suite can check that they survive the metadata mapping. +-- --------------------------------------------------------------------------- +CREATE TABLE log_basic ( + id INT COMMENT 'row id', + name STRING COMMENT 'row name', + price DECIMAL(10, 2) +) COMMENT 'fluss log table for regression' +WITH ( + 'bucket.num' = '3' +); + +INSERT INTO log_basic VALUES + (1, 'alice', CAST(10.10 AS DECIMAL(10, 2))), + (2, 'bob', CAST(20.20 AS DECIMAL(10, 2))), + (3, 'carol', CAST(30.30 AS DECIMAL(10, 2))); + +-- --------------------------------------------------------------------------- +-- log_types: one column per fluss data type that the connector maps, plus an +-- all-NULL row. TIME is deliberately absent: it has no Doris counterpart and +-- gets its own negative fixture later. +-- --------------------------------------------------------------------------- +CREATE TABLE log_types ( + id INT, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ), + ( + 2, + FALSE, + CAST(-1 AS TINYINT), + CAST(-2 AS SMALLINT), + -3, + CAST(-4 AS BIGINT), + CAST(-1.5 AS FLOAT), + CAST(-2.5 AS DOUBLE), + CAST(-123.4567 AS DECIMAL(20, 4)), + CAST('char2' AS CHAR(5)), + 'string2', + CAST(X'040506' AS BINARY(3)), + CAST(X'0c0d' AS BYTES), + DATE '2026-01-02', + TIMESTAMP '2026-01-02 01:02:03.456789', + CAST(TIMESTAMP '2026-01-02 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[4, 5], + MAP['k3', 3], + CAST(ROW(2, 'nested2') AS ROW) + ), + ( + 3, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); + +-- --------------------------------------------------------------------------- +-- log_part: partitioned append-only table. Partitions are created on write +-- (fluss dynamic partitioning), so the partition set is exactly the one below. +-- --------------------------------------------------------------------------- +CREATE TABLE log_part ( + id INT, + name STRING, + dt STRING +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO log_part VALUES + (1, 'p1a', '20260101'), + (2, 'p1b', '20260101'), + (3, 'p2a', '20260102'), + (4, 'p3a', '20260103'); + +-- --------------------------------------------------------------------------- +-- pk_basic: primary-key table. Row 2 is updated and row 3 deleted, so a +-- correct read returns the merged view, not the raw change log. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_basic ( + id INT NOT NULL, + name STRING, + score DOUBLE, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3' +); + +INSERT INTO pk_basic VALUES + (1, 'k1', CAST(1.5 AS DOUBLE)), + (2, 'k2', CAST(2.5 AS DOUBLE)), + (3, 'k3', CAST(3.5 AS DOUBLE)), + (4, 'k4', CAST(4.5 AS DOUBLE)); + +INSERT INTO pk_basic VALUES + (2, 'k2-updated', CAST(22.5 AS DOUBLE)); + +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM pk_basic WHERE id = 3; +SET 'execution.runtime-mode' = 'streaming'; + +-- --------------------------------------------------------------------------- +-- pk_types: same type coverage as log_types, but stored in the kv (compacted) +-- row format that primary-key tables use. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_types ( + id INT NOT NULL, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO pk_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ), + ( + 2, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 4ac5edb4a3322c..ad1fc501e2525b 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -76,12 +76,12 @@ Usage: $0 --hive-modules comma separated hive modules to refresh All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio,fluss " exit 1 } DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,hudi,db2,oceanbase,kerberos,minio" -ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris" +ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris,fluss" COMPONENTS=$2 HELP=0 STOP=0 @@ -257,6 +257,7 @@ RUN_KERBEROS=0 RUN_MINIO=0 RUN_RANGER=0 RUN_POLARIS=0 +RUN_FLUSS=0 RESERVED_PORTS="65535" @@ -303,6 +304,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_RANGER=1 elif [[ "${element}"x == "polaris"x ]]; then RUN_POLARIS=1 + elif [[ "${element}"x == "fluss"x ]]; then + RUN_FLUSS=1 else echo "Invalid component: ${element}" usage @@ -1476,6 +1479,39 @@ start_mariadb() { "${ROOT}/docker-compose/mariadb/data" } +start_fluss() { + local fluss_dir="${ROOT}/docker-compose/fluss" + + # The compose file bind mounts remote.data.dir at the same absolute path it + # uses inside the containers, so Doris BE (running on the host) can read the + # kv snapshots and remote log segments the servers write there. + export FLUSS_COMPOSE_DIR="${fluss_dir}" + envsubst <"${fluss_dir}/fluss.env.tpl" >"${fluss_dir}/fluss.env" + set -a + # shellcheck source=/dev/null + . "${fluss_dir}/fluss.env" + set +a + + render_uid_template "${fluss_dir}/fluss.yaml.tpl" "${fluss_dir}/fluss.yaml" + register_stack_metadata "fluss" "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" + compose_down_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" --remove-orphans + + if [[ "${STOP}" -eq 1 ]]; then + return 0 + fi + + # Fluss 1.0 is unreleased: both images are built from a local checkout. + FLUSS_DOCKER_REUSE_IMAGES="${FLUSS_DOCKER_REUSE_IMAGES:-1}" \ + bash "${fluss_dir}/build-images.sh" + + reset_data_dirs "${FLUSS_REMOTE_DATA_DIR}" + # The fluss image runs as uid 9999, the host directory is created by root. + sudo chmod 777 "${FLUSS_REMOTE_DATA_DIR}" + sudo chmod +x "${fluss_dir}/scripts/run-init-sql.sh" + + compose_up_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" -d --wait +} + start_lakesoul() { echo "RUN_LAKESOUL" cp "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml.tpl "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml @@ -1818,6 +1854,10 @@ if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then launch_component "lakesoul" "${LOG_ROOT}/start_lakesoul.log" start_lakesoul fi +if [[ "${RUN_FLUSS}" -eq 1 ]]; then + launch_component "fluss" "${LOG_ROOT}/start_fluss.log" start_fluss +fi + if [[ "${RUN_MINIO}" -eq 1 ]]; then launch_component "minio" "${LOG_ROOT}/start_minio.log" start_minio fi diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index afbe0b1a5c3ba0..19eda7bf560544 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -161,6 +161,12 @@ hive3PgPort=5732 enableKafkaTest=false kafka_port=19193 +// fluss catalog test config +// to enable fluss test, you need firstly to start fluss containers +// See `docker/thirdparties/run-thirdparties-docker.sh -c fluss` +enableFlussTest=false +fluss_coordinator_port=19123 + // elasticsearch catalog test config // See `docker/thirdparties/run-thirdparties-docker.sh` enableEsTest=false diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy new file mode 100644 index 00000000000000..fb280546a900c4 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Catalog-level checks for the fluss connector: what a user sees before any +// data is read. Fixtures come from +// docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_catalog", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_catalog" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + + // --- the fixture database and its tables are visible ------------------- + // Every result is bound to a variable before it is chained on: in a Groovy + // command expression `sql """..."""​.collect {}` would collect over the + // string, not over the rows. + def databaseRows = sql """show databases from ${catalogName}""" + def databases = databaseRows.collect { it[0] } + assertTrue(databases.contains("fluss_test"), + "fluss_test missing from ${catalogName}: ${databases}") + + sql """switch ${catalogName}""" + sql """use fluss_test""" + + def tableRows = sql """show tables""" + def tables = tableRows.collect { it[0] } + for (String expected : ["log_basic", "log_types", "log_part", "pk_basic", "pk_types"]) { + assertTrue(tables.contains(expected), "table ${expected} missing: ${tables}") + } + + // --- schema mapping ---------------------------------------------------- + // desc rows are [Field, Type, Null, Key, Default, Extra]. + def descLogBasic = sql """desc log_basic""" + assertEquals(["id", "name", "price"], descLogBasic.collect { it[0] }) + + // One column per fluss type the connector maps, plus the id column. A + // dropped or duplicated column shows up here before any query runs. + def descLogTypes = sql """desc log_types""" + assertEquals(["id", "f_boolean", "f_tinyint", "f_smallint", "f_int", "f_bigint", + "f_float", "f_double", "f_decimal", "f_char", "f_string", "f_binary", + "f_bytes", "f_date", "f_timestamp", "f_timestamp_ltz", "f_array", + "f_map", "f_row"], + descLogTypes.collect { it[0] }) + + // The partition key is an ordinary column of the table, not a hidden one. + def descLogPart = sql """desc log_part""" + assertEquals(["id", "name", "dt"], descLogPart.collect { it[0] }) + + // Primary-key columns keep their position; the connector reports every + // column as a key column, which is how Doris models external tables. + def descPkBasic = sql """desc pk_basic""" + assertEquals(["id", "name", "score"], descPkBasic.collect { it[0] }) + + // --- comments survive the metadata mapping ----------------------------- + // Column comments live on the fluss schema, not on the row type: reading + // the row type instead would silently drop every one of them. + def createTableRows = sql """show create table log_basic""" + def createTable = createTableRows[0][1].toString() + assertTrue(createTable.contains("row id"), "column comment lost: ${createTable}") + assertTrue(createTable.contains("fluss log table for regression"), + "table comment lost: ${createTable}") + + // --- refresh keeps the catalog usable ---------------------------------- + sql """refresh catalog ${catalogName}""" + def refreshedRows = sql """show tables""" + def tablesAfterRefresh = refreshedRows.collect { it[0] } + assertEquals(tables.sort(), tablesAfterRefresh.sort()) + + sql """switch internal""" + sql """drop catalog ${catalogName}""" + + // --- property validation happens at catalog creation ------------------- + test { + sql """ + create catalog test_fluss_no_bootstrap properties ( + "type" = "fluss" + ); + """ + exception "Required property 'fluss.bootstrap.servers' is missing" + } + + test { + sql """ + create catalog test_fluss_bad_port properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${externalEnvIp}:not-a-port" + ); + """ + exception "expected a number between 1 and 65535" + } + + test { + sql """ + create catalog test_fluss_bad_union_mode properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "sometimes" + ); + """ + exception "expected one of auto, required, disabled" + } +} From fbf1049e3d34cbbb6e7b967cf29c0cf732cf9f39 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 20:52:22 +0800 Subject: [PATCH 05/35] [feat](fluss) Carry a fluss scan range to BE A fluss scan range is one bucket of one partition, read as a log range, a full primary-key range, or a primary-key range unioned with the lake. This adds the range itself and the two thrift maps that carry it. The payload is an untyped string map rather than a thrift struct of its own. The C++ layer holds no fluss logic -- it hands the map straight to the java scanner in be-java-extensions -- so a struct would only add a field-by-field transcription in the middle, which is what paimon pays for in paimon_jni_reader.cpp. es_params and jdbc_params are the same shape for the same reason, and it keeps a plugin's parameters out of the core IDL. Only what varies per split rides in the range. Bootstrap servers, table identity and the client and table options are the same for every split of a scan, so they go once into TFileScanRangeParams.fluss_properties, next to the paimon and ES properties already there for that reason; a table with 100 partitions and 128 buckets would otherwise serialize them 12800 times. Nothing populates the scan-level map yet -- the split planner does, in the next change. Partition columns are declared to the engine rather than returned by the scanner. That is forced by the union read: the lake half of a union is paimon ranges, the paimon connector already declares its partition keys that way, and the file-slot / partition-slot split is decided once per scan node. Co-Authored-By: Claude Opus 5 (1M context) --- .../doris/connector/fluss/FlussScanRange.java | 294 ++++++++++++++++ .../connector/fluss/FlussScanRangeTest.java | 328 ++++++++++++++++++ gensrc/thrift/PlanNodes.thrift | 11 + 3 files changed, 633 insertions(+) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java new file mode 100644 index 00000000000000..a07964afca51e0 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * One unit of fluss reading: a single bucket of a single partition, read one of three ways. + * + *

The payload is an untyped string map rather than a thrift struct of its own. The C++ layer holds + * no fluss logic — it hands the map straight to the java scanner in be-java-extensions — so a typed + * struct would buy nothing but a transcription step in the middle (which is what paimon pays for, + * {@code paimon_jni_reader.cpp} rebuilding a string map field by field). {@code es_params} and + * {@code jdbc_params} are the same shape for the same reason. + * + *

Only what VARIES per split lives here. Bootstrap servers, table identity, client and table + * options and the projection are identical for every split of a scan, so they are written once at + * scan-node level ({@code TFileScanRangeParams.fluss_properties}); a table with 100 partitions and + * 128 buckets would otherwise serialize them 12800 times. + * + *

Partition columns are NOT part of what the scanner returns. The connector declares the partition + * keys as {@code path_partition_keys}, which makes the engine leave them out of the scanner's + * projection and materialize them from {@link #getPartitionValues()} instead. That is forced by the + * union read: the lake half of a union comes from paimon ranges, and the paimon connector already + * declares its partition keys that way — the file-slot / partition-slot split is decided once per scan + * node, so both halves have to agree on it. + */ +public class FlussScanRange implements ConnectorScanRange { + + private static final long serialVersionUID = 1L; + + /** How the scanner reads this range; the wire value is the enum name. */ + public enum RangeType { + /** Log-only read of one bucket over {@code [logStart, logStop)}. */ + LOG, + /** Full read of one bucket of a primary-key table: kv snapshot plus the log after it. */ + PK_FULL, + /** Primary-key read merging the lake's splits for one bucket with the log tail after them. */ + UNION_PK + } + + public static final String PROP_RANGE_TYPE = "fluss.range_type"; + public static final String PROP_PARTITION_ID = "fluss.partition_id"; + public static final String PROP_PARTITION_NAME = "fluss.partition_name"; + public static final String PROP_BUCKET_ID = "fluss.bucket_id"; + public static final String PROP_LOG_START_OFFSET = "fluss.log_start_offset"; + public static final String PROP_LOG_STOP_OFFSET = "fluss.log_stop_offset"; + public static final String PROP_KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; + public static final String PROP_LAKE_SNAPSHOT_ID = "fluss.lake_snapshot_id"; + public static final String PROP_LAKE_SPLITS = "fluss.lake_splits"; + + /** {@code kv_snapshot_id} for a bucket that has never been snapshotted. */ + public static final long NO_KV_SNAPSHOT = -1L; + + /** + * Separator for {@link #PROP_LAKE_SPLITS}. Each element is base64, whose alphabet + * ({@code A-Za-z0-9+/=}) has no comma, so joining is unambiguous. The factory rejects any element + * that contains one rather than trusting the caller. + */ + private static final String LAKE_SPLIT_SEPARATOR = ","; + + private final RangeType rangeType; + private final Partition partition; + private final int bucketId; + private final Map properties; + + private FlussScanRange(RangeType rangeType, Partition partition, int bucketId, + Map properties) { + this.rangeType = rangeType; + this.partition = partition; + this.bucketId = bucketId; + this.properties = Collections.unmodifiableMap(properties); + } + + /** + * A log range over {@code [logStartOffset, logStopOffset)} of one bucket. {@code logStartOffset} is + * either a real offset or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel, passed through + * verbatim — the scanner hands it back to fluss, which is the only side that interprets it. + */ + public static FlussScanRange log(Partition partition, int bucketId, + long logStartOffset, long logStopOffset) { + Map props = baseProps(RangeType.LOG, partition, bucketId, + logStartOffset, logStopOffset); + return new FlussScanRange(RangeType.LOG, partition, bucketId, props); + } + + /** + * A full primary-key range: the kv snapshot {@code kvSnapshotId} plus the log from + * {@code logStartOffset} (the offset that snapshot was taken at) up to {@code logStopOffset}. + * {@code kvSnapshotId} is {@link #NO_KV_SNAPSHOT} when the bucket has never been snapshotted, in + * which case the whole state is replayed from the log. + */ + public static FlussScanRange pkFull(Partition partition, int bucketId, long kvSnapshotId, + long logStartOffset, long logStopOffset) { + Map props = baseProps(RangeType.PK_FULL, partition, bucketId, + logStartOffset, logStopOffset); + props.put(PROP_KV_SNAPSHOT_ID, String.valueOf(kvSnapshotId)); + return new FlussScanRange(RangeType.PK_FULL, partition, bucketId, props); + } + + /** + * A union primary-key range: this bucket's splits of lake snapshot {@code lakeSnapshotId}, merged + * with the log from {@code logStartOffset} (where that lake snapshot ended, exclusive) up to + * {@code logStopOffset}. Each element of {@code lakeSplits} is a base64-encoded serialized lake + * split. + */ + public static FlussScanRange unionPk(Partition partition, int bucketId, long lakeSnapshotId, + List lakeSplits, long logStartOffset, long logStopOffset) { + Objects.requireNonNull(lakeSplits, "lakeSplits"); + if (lakeSplits.isEmpty()) { + throw new IllegalArgumentException( + "a UNION_PK range needs at least one lake split; a bucket with none is a plain log read"); + } + for (String split : lakeSplits) { + if (split == null || split.contains(LAKE_SPLIT_SEPARATOR)) { + // Would silently split one entry into two on the scanner side. + throw new IllegalArgumentException( + "lake split is not base64 (null or contains '" + LAKE_SPLIT_SEPARATOR + "'): " + split); + } + } + Map props = baseProps(RangeType.UNION_PK, partition, bucketId, + logStartOffset, logStopOffset); + props.put(PROP_LAKE_SNAPSHOT_ID, String.valueOf(lakeSnapshotId)); + props.put(PROP_LAKE_SPLITS, String.join(LAKE_SPLIT_SEPARATOR, lakeSplits)); + return new FlussScanRange(RangeType.UNION_PK, partition, bucketId, props); + } + + private static Map baseProps(RangeType rangeType, Partition partition, + int bucketId, long logStartOffset, long logStopOffset) { + Objects.requireNonNull(partition, "partition"); + Map props = new LinkedHashMap<>(); + props.put(PROP_RANGE_TYPE, rangeType.name()); + if (partition.isPartitioned()) { + props.put(PROP_PARTITION_ID, String.valueOf(partition.id)); + props.put(PROP_PARTITION_NAME, partition.name); + } + props.put(PROP_BUCKET_ID, String.valueOf(bucketId)); + props.put(PROP_LOG_START_OFFSET, String.valueOf(logStartOffset)); + props.put(PROP_LOG_STOP_OFFSET, String.valueOf(logStopOffset)); + return props; + } + + public RangeType getRangeType() { + return rangeType; + } + + public Partition getPartition() { + return partition; + } + + @Override + public String getTableFormatType() { + return "fluss"; + } + + @Override + public Map getProperties() { + return properties; + } + + @Override + public Map getPartitionValues() { + return partition.values; + } + + /** + * Always true: a fluss range's partition values come from fluss metadata, never from a file path. + * On an unpartitioned table that keeps the engine from falling back to parsing a path that does not + * exist (the range has no path at all). + */ + @Override + public boolean isPartitionBearing() { + return true; + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setFlussParams(new LinkedHashMap<>(properties)); + Map partitionValues = partition.values; + if (!partitionValues.isEmpty()) { + List keys = new ArrayList<>(partitionValues.size()); + List values = new ArrayList<>(partitionValues.size()); + List isNull = new ArrayList<>(partitionValues.size()); + for (Map.Entry entry : partitionValues.entrySet()) { + keys.add(entry.getKey()); + values.add(entry.getValue()); + // Never null: fluss rejects a null partition value at write time + // (PartitionGetter.getResolvedPartitionSpec), so there is no null sentinel to decode. + isNull.add(false); + } + rangeDesc.setColumnsFromPathKeys(keys); + rangeDesc.setColumnsFromPath(values); + rangeDesc.setColumnsFromPathIsNull(isNull); + } + } + + @Override + public String toString() { + return "FlussScanRange{" + rangeType + + (partition.isPartitioned() ? ", partition=" + partition.name : "") + + ", bucket=" + bucketId + + ", " + properties + "}"; + } + + /** + * Which partition a range belongs to, as the three facts that must agree: fluss's partition id + * (what the scanner subscribes by), its Hive-style {@code k=v/k=v} name (what the user sees) and + * the per-column values (what the engine materializes the partition columns from). Bundled so an + * unpartitioned table cannot be described half-way — {@link #NONE} is the only way to say "none". + */ + public static final class Partition implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + + /** The single pseudo-partition of an unpartitioned table. */ + public static final Partition NONE = new Partition(-1L, null, Collections.emptyMap()); + + private final long id; + private final String name; + private final Map values; + + private Partition(long id, String name, Map values) { + this.id = id; + this.name = name; + this.values = values; + } + + /** + * @param id fluss's partition id + * @param name the Hive-style {@code k=v/k=v} partition name Doris uses + * @param values partition column name to value, in partition-key order + */ + public static Partition of(long id, String name, Map values) { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(values, "values"); + if (values.isEmpty()) { + throw new IllegalArgumentException( + "partition " + name + " has no column values; use Partition.NONE for an " + + "unpartitioned table"); + } + return new Partition(id, name, Collections.unmodifiableMap(new LinkedHashMap<>(values))); + } + + /** + * Whether this is a real partition rather than {@link #NONE}. Asked of the field, not of + * object identity: a range crosses a java-serialization boundary, which would hand back a + * copy of {@code NONE} that {@code ==} no longer recognizes. + */ + public boolean isPartitioned() { + return name != null; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public Map getValues() { + return values; + } + + @Override + public String toString() { + return this == NONE ? "Partition.NONE" : name + "(" + id + ")"; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java new file mode 100644 index 00000000000000..21ee674fec7d04 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java @@ -0,0 +1,328 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The range is the FE half of the wire contract with the be-java-extensions scanner: an untyped string + * map that nothing between here and there type-checks. So these assert the map WHOLE — every key, its + * exact rendering, and the absence of keys that do not belong to the range type — rather than spot + * checking. A key that is quietly renamed, dropped or leaked across range types is exactly the failure + * this file exists to catch. + */ +public class FlussScanRangeTest { + + private static final FlussScanRange.Partition DT_20260101 = + FlussScanRange.Partition.of(77L, "dt=20260101", + Collections.singletonMap("dt", "20260101")); + + @Test + public void logRangeCarriesOffsetsAndNothingElse() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 3, 10L, 42L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "LOG"); + expected.put("fluss.bucket_id", "3"); + expected.put("fluss.log_start_offset", "10"); + expected.put("fluss.log_stop_offset", "42"); + Assertions.assertEquals(expected, range.getProperties()); + Assertions.assertEquals(FlussScanRange.RangeType.LOG, range.getRangeType()); + } + + @Test + public void pkFullRangeCarriesTheKvSnapshotButNoLakeFields() { + FlussScanRange range = + FlussScanRange.pkFull(FlussScanRange.Partition.NONE, 0, 900L, 120L, 500L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "PK_FULL"); + expected.put("fluss.bucket_id", "0"); + expected.put("fluss.log_start_offset", "120"); + expected.put("fluss.log_stop_offset", "500"); + expected.put("fluss.kv_snapshot_id", "900"); + Assertions.assertEquals(expected, range.getProperties()); + } + + /** + * A bucket that has never been snapshotted still produces a range: the scanner replays its whole + * state from the log. The sentinel has to survive as {@code -1}, because "key missing" and + * "no snapshot" would otherwise be indistinguishable on the scanner side. + */ + @Test + public void pkFullWithoutASnapshotSendsTheSentinel() { + FlussScanRange range = FlussScanRange.pkFull(FlussScanRange.Partition.NONE, 0, + FlussScanRange.NO_KV_SNAPSHOT, LogScanner.EARLIEST_OFFSET, 500L); + + Assertions.assertEquals("-1", range.getProperties().get("fluss.kv_snapshot_id")); + Assertions.assertEquals("-2", range.getProperties().get("fluss.log_start_offset")); + } + + /** + * EARLIEST is fluss's own sentinel, not a Doris one: the scanner hands the value straight back to + * fluss. If fluss ever renumbers it, FE and the scanner move together — but the rendering must stay + * the raw number, never a name the scanner would have to translate. + */ + @Test + public void earliestOffsetGoesOutVerbatim() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 1, + LogScanner.EARLIEST_OFFSET, 7L); + + Assertions.assertEquals(String.valueOf(LogScanner.EARLIEST_OFFSET), + range.getProperties().get("fluss.log_start_offset")); + Assertions.assertEquals("-2", range.getProperties().get("fluss.log_start_offset")); + } + + @Test + public void unionPkRangeCarriesLakeSplitsButNoKvSnapshot() { + FlussScanRange range = FlussScanRange.unionPk(DT_20260101, 2, 41L, + Arrays.asList("c3BsaXQtMQ==", "c3BsaXQtMg=="), 300L, 305L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "UNION_PK"); + expected.put("fluss.partition_id", "77"); + expected.put("fluss.partition_name", "dt=20260101"); + expected.put("fluss.bucket_id", "2"); + expected.put("fluss.log_start_offset", "300"); + expected.put("fluss.log_stop_offset", "305"); + expected.put("fluss.lake_snapshot_id", "41"); + expected.put("fluss.lake_splits", "c3BsaXQtMQ==,c3BsaXQtMg=="); + Assertions.assertEquals(expected, range.getProperties()); + } + + /** + * The lake splits ride in one string, comma separated, which is only unambiguous because base64 + * has no comma. This pins the premise rather than assuming it: every byte value, encoded, must + * stay comma-free. + */ + @Test + public void base64NeverContainsTheLakeSplitSeparator() { + byte[] allBytes = new byte[256]; + for (int i = 0; i < 256; i++) { + allBytes[i] = (byte) i; + } + for (int start = 0; start < 256; start++) { + String encoded = Base64.getEncoder() + .encodeToString(Arrays.copyOfRange(allBytes, start, 256)); + Assertions.assertFalse(encoded.contains(","), + "base64 of bytes " + start + ".. contains a comma: " + encoded); + } + } + + /** A split that could be mis-split on the scanner side must fail here, not read half a split there. */ + @Test + public void lakeSplitContainingTheSeparatorIsRejected() { + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.unionPk(DT_20260101, 0, 1L, + Arrays.asList("ok", "not,base64"), 0L, 1L)); + Assertions.assertTrue(e.getMessage().contains("not,base64"), e.getMessage()); + } + + /** A union range with nothing from the lake is a plain log read; producing one would double-read. */ + @Test + public void unionPkWithoutLakeSplitsIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.unionPk(DT_20260101, 0, 1L, Collections.emptyList(), 0L, 1L)); + } + + @Test + public void unpartitionedRangeOmitsThePartitionKeys() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L); + + Assertions.assertFalse(range.getProperties().containsKey("fluss.partition_id")); + Assertions.assertFalse(range.getProperties().containsKey("fluss.partition_name")); + Assertions.assertTrue(range.getPartitionValues().isEmpty()); + } + + /** + * An unpartitioned range still reports itself partition-bearing: the engine reads a null partition + * value list as "parse the values out of the file path", and a fluss range has no path to parse. + */ + @Test + public void rangesAreAlwaysPartitionBearing() { + Assertions.assertTrue( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L).isPartitionBearing()); + Assertions.assertTrue( + FlussScanRange.log(DT_20260101, 0, 0L, 1L).isPartitionBearing()); + } + + @Test + public void tableFormatTypeSelectsTheFlussReader() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L); + + Assertions.assertEquals("fluss", range.getTableFormatType()); + // Default of the SPI: everything fluss reads goes through the JNI scanner, there is no + // native path to downgrade to. + Assertions.assertEquals("jni", range.getFileFormat()); + } + + @Test + public void populateRangeParamsWritesTheWholeMapIntoFlussParams() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 5, 0L, 9L); + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + range.populateRangeParams(formatDesc, rangeDesc); + + Assertions.assertEquals(range.getProperties(), formatDesc.getFlussParams()); + // No other format's params may be touched — BE dispatches on table_format_type and would + // read a stale struct if one were half-filled. + Assertions.assertFalse(formatDesc.isSetPaimonParams()); + Assertions.assertFalse(formatDesc.isSetIcebergParams()); + Assertions.assertFalse(formatDesc.isSetJdbcParams()); + Assertions.assertFalse(formatDesc.isSetEsParams()); + // Unpartitioned: nothing for BE to materialize from the range. + Assertions.assertFalse(rangeDesc.isSetColumnsFromPathKeys()); + Assertions.assertFalse(rangeDesc.isSetColumnsFromPath()); + } + + /** + * Partition columns are not in what the scanner returns — the connector declares them as + * path partition keys — so BE materializes them from here. Order matters: BE pairs the value list + * with the key list positionally. + */ + @Test + public void populateRangeParamsHandsPartitionColumnsToBe() { + Map values = new LinkedHashMap<>(); + values.put("dt", "20260101"); + values.put("region", "cn"); + FlussScanRange range = FlussScanRange.log( + FlussScanRange.Partition.of(9L, "dt=20260101/region=cn", values), 0, 0L, 1L); + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + range.populateRangeParams(formatDesc, rangeDesc); + + Assertions.assertEquals(Arrays.asList("dt", "region"), rangeDesc.getColumnsFromPathKeys()); + Assertions.assertEquals(Arrays.asList("20260101", "cn"), rangeDesc.getColumnsFromPath()); + // fluss refuses a null partition value at write time, so there is no null to signal. + Assertions.assertEquals(Arrays.asList(false, false), rangeDesc.getColumnsFromPathIsNull()); + } + + /** Half-describing a partition is the bug this rules out; NONE is the only way to say "none". */ + @Test + public void partitionWithoutColumnValuesIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.Partition.of(1L, "dt=20260101", Collections.emptyMap())); + } + + /** + * The SPI declares a scan range {@link java.io.Serializable} and the engine takes it at its word. + * Round-tripping also pins that "is this partitioned?" survives — answering it by identity against + * the NONE singleton would come back wrong on the far side of a deserialization. + */ + @Test + public void rangeSurvivesJavaSerialization() throws Exception { + for (FlussScanRange original : Arrays.asList( + FlussScanRange.log(FlussScanRange.Partition.NONE, 3, 10L, 42L), + FlussScanRange.unionPk(DT_20260101, 2, 41L, + Collections.singletonList("c3BsaXQ="), 300L, 305L))) { + FlussScanRange restored = roundTrip(original); + + Assertions.assertEquals(original.getProperties(), restored.getProperties()); + Assertions.assertEquals(original.getPartitionValues(), restored.getPartitionValues()); + Assertions.assertEquals(original.getRangeType(), restored.getRangeType()); + + // The restored NONE is a different object than the singleton; the range must still be + // able to tell it is unpartitioned. + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + restored.populateRangeParams(formatDesc, rangeDesc); + Assertions.assertEquals(original.getProperties(), formatDesc.getFlussParams()); + } + } + + /** + * {@code NONE} is a singleton but has no {@code readResolve}, so deserializing gives back a + * different object. Anything that answers "is this partitioned?" by comparing against the singleton + * would say yes here, and the range would then claim a partition id of -1 with no columns. + */ + @Test + public void restoredNonePartitionStillReportsUnpartitioned() throws Exception { + FlussScanRange restored = roundTrip( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L)); + + Assertions.assertNotSame(FlussScanRange.Partition.NONE, restored.getPartition()); + Assertions.assertFalse(restored.getPartition().isPartitioned()); + Assertions.assertTrue(restored.getPartitionValues().isEmpty()); + } + + @Test + public void partitionExposesItsIdentity() { + Assertions.assertTrue(DT_20260101.isPartitioned()); + Assertions.assertEquals(77L, DT_20260101.getId()); + Assertions.assertEquals("dt=20260101", DT_20260101.getName()); + Assertions.assertEquals(Collections.singletonMap("dt", "20260101"), DT_20260101.getValues()); + Assertions.assertFalse(FlussScanRange.Partition.NONE.isPartitioned()); + } + + private static FlussScanRange roundTrip(FlussScanRange range) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(range); + } + try (ObjectInputStream in = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (FlussScanRange) in.readObject(); + } + } + + /** Guards the fixture itself: an unmodifiable view is still a live view of a caller's map. */ + @Test + public void partitionValuesAreCopiedFromTheCaller() { + Map mutable = new LinkedHashMap<>(); + mutable.put("dt", "20260101"); + FlussScanRange.Partition partition = FlussScanRange.Partition.of(1L, "dt=20260101", mutable); + + mutable.put("dt", "20260102"); + + Assertions.assertEquals("20260101", partition.getValues().get("dt")); + } + + /** The property map is the wire payload; a caller must not be able to edit it after the fact. */ + @Test + public void propertiesAreImmutable() { + List ranges = Arrays.asList( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L), + FlussScanRange.pkFull(DT_20260101, 0, 1L, 0L, 1L), + FlussScanRange.unionPk(DT_20260101, 0, 1L, + Collections.singletonList("c3BsaXQ="), 0L, 1L)); + for (FlussScanRange range : ranges) { + Assertions.assertThrows(UnsupportedOperationException.class, + () -> range.getProperties().put("fluss.bucket_id", "999")); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> range.getPartitionValues().put("dt", "x")); + } + } +} diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index bbf2c64b05d77e..6dd1849cc5a9ce 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -478,6 +478,14 @@ struct TTableFormatFileDesc { // ES per-shard parameters (used when table_format_type == "es") // Contains: index, type, shard_id, host_port, es_hosts 13: optional map es_params + // Fluss per-split parameters (used when table_format_type == "fluss"). + // Carries ONLY what varies per split: partition/bucket identity, range type, log offsets, + // kv/lake snapshot ids and lake splits. Everything constant for the whole scan (bootstrap + // servers, table identity, client/table options) lives in TFileScanRangeParams.fluss_properties + // so it is not re-serialized once per bucket. + // Untyped on purpose: BE C++ holds no fluss logic, it hands this map straight to the Java + // scanner, so a typed struct would only add a transcription step (see es_params, jdbc_params). + 14: optional map fluss_params } // Deprecated, hive text talbe is a special format, not a serde type @@ -564,6 +572,9 @@ struct TFileScanRangeParams { // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and // logical initial-default materialization. 34: optional i32 iceberg_scan_semantics_version + // Fluss scan-level properties (bootstrap servers, table identity, client/table options, + // projected columns). Set at ScanNode level to avoid redundant serialization in each split. + 35: optional map fluss_properties } struct TFileRangeDesc { From 67d8e72562740668fd1742d40e40eb35031d21a8 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 21:14:07 +0800 Subject: [PATCH 06/35] [feat](fluss) Plan the scan of a fluss log table Reads a log table one bucket at a time, from the earliest offset fluss still holds up to the offset the log had reached when planning ran. That stopping offset is taken once per partition so every bucket of a partition stops at the same view of the table; a bucket nobody has written to yields no range at all. Partition naming moves into FlussPartitions, shared with the metadata listing. The pruned partition names the engine hands to planning are the ones that listing produced, so the two have to render them identically -- rendered separately, planning matches none of them and scans nothing, which looks like an empty table rather than a bug. What this cannot serve yet it refuses by name: a primary-key table, and a table whose tiering has committed to the lake. Reading the latter as fluss-only would return whatever the log still holds and drop everything already tiered away -- a query that succeeds with rows missing. fluss.union_read.mode=disabled is how a user asks for that read on purpose, and a lake table that has never tiered still falls back to the log alone, because then the log is the whole table. Planning only reads metadata, so it is safe under EXPLAIN, which does reach planScan. The EXPLAIN line it appends is what a regression test has to read to tell a union read from a silent fallback to a fluss-only one. The cluster tests that write rows need --add-opens java.nio for Arrow, fluss's default log format; the BE JVM that will host the scanner already gets that flag from bin/start_be.sh. Co-Authored-By: Claude Opus 5 (1M context) --- fe/fe-connector/fe-connector-fluss/pom.xml | 15 + .../doris/connector/fluss/FlussConnector.java | 10 + .../fluss/FlussConnectorMetadata.java | 30 +- .../connector/fluss/FlussPartitions.java | 63 +++ .../fluss/FlussScanPlanProvider.java | 264 +++++++++++ .../fluss/FlussLogScanPlanClusterTest.java | 279 +++++++++++ .../connector/fluss/FlussSplitPlanTest.java | 442 ++++++++++++++++++ .../fluss/RecordingFlussAdminOps.java | 32 +- 8 files changed, 1110 insertions(+), 25 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml index 168c64cec8efb8..4ecf7e35eb830c 100644 --- a/fe/fe-connector/fe-connector-fluss/pom.xml +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -141,6 +141,21 @@ under the License. doris-fe-connector-fluss + + org.apache.maven.plugins + maven-surefire-plugin + + + --add-opens=java.base/java.nio=ALL-UNNAMED + + maven-assembly-plugin diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 02aa7c7c10cf38..82b39712ecd5b2 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -22,6 +22,7 @@ import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorTestResult; import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.fluss.client.Connection; @@ -61,6 +62,15 @@ public ConnectorMetadata getMetadata(ConnectorSession session) { FlussConnectorProperties.typeMappingOptions(properties)); } + /** + * A fresh provider per call, which is what the engine wants: it memoizes one instance per scan node + * and that instance keeps the just-planned range counts for the node's EXPLAIN line. + */ + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return new FlussScanPlanProvider(adminOps(), properties); + } + @Override public ConnectorTestResult testConnection(ConnectorSession session) { try { diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 3b37775477f075..82b87b349e7ad4 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -163,11 +163,9 @@ public List listPartitionNames(ConnectorSession session, ConnectorTableH /** * Lists the table's partitions in the Hive-style {@code k1=v1/k2=v2} naming every Doris catalog * uses, which is not fluss's own {@code v1$v2} spelling — fe-core parses the segments back out of - * the name for {@code SHOW PARTITIONS} and the {@code partition_values} function. No escaping is - * needed on the way: fluss rejects a partition value that is not ASCII alphanumerics, {@code _} or - * {@code -} (TablePath#detectInvalidName, applied by PartitionUtils#validatePartitionValues), so a - * value can contain neither the {@code =} nor the {@code /} that would make two partitions render - * to one name. That same rule is why no value can be SQL NULL and the null-flag list stays empty. + * the name for {@code SHOW PARTITIONS} and the {@code partition_values} function. The rendering is + * {@link FlussPartitions}', shared with split planning: the pruned names the engine derives from + * this listing are what planning matches its partitions against, so one renderer or none. * *

{@code filter} is ignored: server-side partition pruning exists in fluss (a partial * {@code PartitionSpec}) but it takes a spec, not a predicate, and the predicate-to-spec reduction @@ -188,24 +186,12 @@ public List listPartitions(ConnectorSession session, List flussPartitions = adminOps.listPartitionInfos(flussHandle.toTablePath()); List result = new ArrayList<>(flussPartitions.size()); for (PartitionInfo partition : flussPartitions) { - Map spec = partition.getPartitionSpec().getSpecMap(); - // Both lists follow the partition-COLUMN order, not the spec's iteration order, because - // fe-core zips them positionally against the partition columns. - Map values = new LinkedHashMap<>(); - List orderedValues = new ArrayList<>(partitionKeys.size()); - StringBuilder name = new StringBuilder(); - for (String partitionKey : partitionKeys) { - String value = spec.get(partitionKey); - values.put(partitionKey, value); - orderedValues.add(value); - if (name.length() > 0) { - name.append('/'); - } - name.append(partitionKey).append('=').append(value); - } + FlussScanRange.Partition resolved = FlussPartitions.toScanPartition(partition, partitionKeys); + // The values already follow partition-COLUMN order (fe-core zips them positionally against + // the partition columns); the null-flag list stays empty because fluss allows no null value. result.add(new ConnectorPartitionInfo( - name.toString(), values, Collections.emptyMap(), - orderedValues, Collections.emptyList())); + resolved.getName(), resolved.getValues(), Collections.emptyMap(), + new ArrayList<>(resolved.getValues().values()), Collections.emptyList())); } return result; } diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java new file mode 100644 index 00000000000000..ab7b5d5854a523 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.metadata.PartitionInfo; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * How a fluss partition is named on the Doris side — in one place, because two callers must agree on it. + * + *

Fluss names a partition by joining its values ({@code 20260101$cn}); Doris names it the Hive way + * ({@code dt=20260101/region=cn}), which is what fe-core parses back out for {@code SHOW PARTITIONS} + * and what it hands back as the pruned partition set. So the metadata listing and split planning both + * render this name, and if the two renderings ever drifted apart, planning would match none of the + * pruned names and silently scan nothing. + * + *

No escaping is needed: fluss rejects a partition value that is not ASCII alphanumerics, {@code _} + * or {@code -} (TablePath#detectInvalidName via PartitionUtils#validatePartitionValues), so a value can + * contain neither {@code =} nor {@code /}, and cannot be null. + */ +final class FlussPartitions { + + private FlussPartitions() { + } + + /** + * The scan-side view of {@code partition}: fluss's partition id, the Doris partition name and the + * per-column values, in partition-key order (fe-core zips values against the partition columns + * positionally). + */ + static FlussScanRange.Partition toScanPartition(PartitionInfo partition, List partitionKeys) { + Map spec = partition.getPartitionSpec().getSpecMap(); + Map values = new LinkedHashMap<>(); + StringBuilder name = new StringBuilder(); + for (String partitionKey : partitionKeys) { + String value = spec.get(partitionKey); + values.put(partitionKey, value); + if (name.length() > 0) { + name.append('/'); + } + name.append(partitionKey).append('=').append(value); + } + return FlussScanRange.Partition.of(partition.getPartitionId(), name.toString(), values); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java new file mode 100644 index 00000000000000..7f9b05cf550833 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TFileScanRangeParams; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.exception.LakeTableSnapshotNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.TablePath; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Turns a fluss table into the scan ranges that cover it. + * + *

A log table is read one bucket at a time, from the earliest offset fluss still holds up to the + * offset the log had reached when planning ran. That stopping offset is taken once, for all of a + * partition's buckets, so every bucket of a partition stops at the same view of the table; without it + * a bucket planned later would read rows written after the query started while an earlier bucket did + * not. A bucket that has never been written to (stopping offset 0) yields no range at all. + * + *

Planning only reads metadata — partition lists and offsets — so it is safe to run for an + * {@code EXPLAIN}, which does reach {@code planScan}. (There is no explain-only signal on the SPI in + * this branch; the point is that fluss does not need one. A future change that takes a snapshot lease + * during planning would.) + * + *

What is NOT here yet: primary-key tables, and the union of a table's lake with its log. Both are + * refused loudly rather than served as a partial answer — a datalake table read as fluss-only would + * silently return just the rows that have not been tiered away, which looks like a working query. + * {@code fluss.union_read.mode=disabled} is how a user asks for the fluss-only read on purpose. + */ +public class FlussScanPlanProvider implements ConnectorScanPlanProvider { + + /** + * Prefix marking a node property that belongs to BE rather than to the engine. Everything under it + * is copied verbatim into {@code TFileScanRangeParams.fluss_properties}; everything else (the + * engine's own keys, e.g. the partition keys) is not. + */ + static final String BE_PROPERTY_PREFIX = "fluss."; + + /** + * Prefix for the fluss client configuration. The scanner strips it and hands the rest to fluss's + * {@code Configuration}, so the connector never has to enumerate fluss's own option names. + */ + static final String PROP_CLIENT_PREFIX = "fluss.client."; + + static final String PROP_DB_NAME = "fluss.db_name"; + static final String PROP_TABLE_NAME = "fluss.table_name"; + + private final FlussAdminOps adminOps; + private final Map catalogProperties; + + /** + * What the last {@link #planScan} produced, for the EXPLAIN line. Plain fields, not volatile: the + * engine memoizes one provider instance per scan node and plans that node on the FE planning + * thread, then renders EXPLAIN from the same thread afterwards. This connector declares neither + * batch scan nor streaming splits, which are what would move planning off that thread — enabling + * either means revisiting this (the ES provider carries the same caveat). + */ + private int plannedLogRanges; + private boolean plannedUnionRead; + + public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties) { + this.adminOps = adminOps; + this.catalogProperties = catalogProperties; + } + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + FlussTableHandle handle = (FlussTableHandle) request.getTableHandle(); + FlussConnectorProperties.UnionReadMode mode = + FlussConnectorProperties.unionReadMode(catalogProperties); + rejectWhatIsNotImplemented(handle, mode); + + TablePath tablePath = handle.toTablePath(); + List buckets = allBuckets(handle.getBucketCount()); + List ranges = new ArrayList<>(); + + if (handle.isPartitioned()) { + for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { + // fluss's own partition name ("20260101$cn"), not the Doris one: this is a fluss API. + Map stopping = adminOps.listOffsets( + tablePath, partition.getPartitionName(), buckets, new OffsetSpec.LatestSpec()); + appendLogRanges(ranges, + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), + buckets, stopping); + } + } else { + Map stopping = + adminOps.listOffsets(tablePath, buckets, new OffsetSpec.LatestSpec()); + appendLogRanges(ranges, FlussScanRange.Partition.NONE, buckets, stopping); + } + + plannedLogRanges = ranges.size(); + plannedUnionRead = false; + return ranges; + } + + /** + * Refuses the reads this connector cannot serve yet, naming what would have to change. Serving them + * partially is the failure mode to avoid: a datalake table planned as fluss-only returns whatever + * the log still holds and drops everything tiering has already moved into the lake, which is a + * successful query with missing rows. + */ + private void rejectWhatIsNotImplemented(FlussTableHandle handle, + FlussConnectorProperties.UnionReadMode mode) { + if (handle.hasPrimaryKey()) { + throw new DorisConnectorException("Reading the fluss primary-key table '" + + handle.getDatabaseName() + "." + handle.getTableName() + + "' is not supported yet; only log tables can be read."); + } + if (!handle.isDataLakeEnabled() || mode == FlussConnectorProperties.UnionReadMode.DISABLED) { + // Not a lake table, or the user asked for the fluss-only read explicitly. + return; + } + try { + adminOps.getReadableLakeSnapshot(handle.toTablePath()); + } catch (LakeTableSnapshotNotExistException e) { + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' has no readable lake snapshot yet, and '" + + FlussConnectorProperties.UNION_READ_MODE + "=required' forbids falling back to " + + "a fluss-only read. Wait for the tiering service to commit, or set the property " + + "to auto or disabled.", e); + } + // Nothing is in the lake, so the log holds everything: the fluss-only read is the whole table. + return; + } + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' is tiered into a lake and reading it requires combining the " + + "lake with the fluss log, which is not supported yet. Set '" + + FlussConnectorProperties.UNION_READ_MODE + "=disabled' to read only what the fluss log " + + "still holds, which is NOT the whole table."); + } + + /** + * The partitions to scan: those the engine's pruning left, or all of them when it pruned nothing. + * A pruned name that fluss no longer lists is simply absent from the result — the partition was + * dropped between pruning and planning, and there is nothing left to read. + */ + private List selectedPartitions(FlussTableHandle handle, List requiredPartitions) { + List partitions = adminOps.listPartitionInfos(handle.toTablePath()); + if (requiredPartitions.isEmpty()) { + return partitions; + } + Set required = new HashSet<>(requiredPartitions); + List selected = new ArrayList<>(partitions.size()); + for (PartitionInfo partition : partitions) { + // Matched on the DORIS name, which is what the engine pruned over: FlussPartitions renders + // both this and the metadata listing the engine pruned, so the two cannot disagree. + if (required.contains( + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()).getName())) { + selected.add(partition); + } + } + return selected; + } + + private static void appendLogRanges(List ranges, + FlussScanRange.Partition partition, List buckets, Map stopping) { + for (int bucket : buckets) { + Long stop = stopping.get(bucket); + if (stop == null || stop <= 0) { + // Never written to: no range at all rather than an empty one for BE to open and close. + // A bucket whose records have all aged out of the log is NOT this case — its latest + // offset stayed where it was — and still gets a range that reads nothing, which costs + // one scanner rather than an extra round trip for every bucket to find out. + continue; + } + ranges.add(FlussScanRange.log(partition, bucket, LogScanner.EARLIEST_OFFSET, stop)); + } + } + + private static List allBuckets(int bucketCount) { + List buckets = new ArrayList<>(bucketCount); + for (int bucket = 0; bucket < bucketCount; bucket++) { + buckets.add(bucket); + } + return buckets; + } + + /** + * What every range of this scan shares. The {@code fluss.}-prefixed entries are the BE half and are + * forwarded by {@link #populateScanLevelParams}; {@code path_partition_keys} is the engine's, and + * declaring it is what keeps the partition columns out of the scanner's projection so BE + * materializes them from each range instead (see {@link FlussScanRange}). + */ + @Override + public Map getScanNodeProperties(ConnectorSession session, ConnectorTableHandle handle, + List columns, Optional filter) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + Map props = new LinkedHashMap<>(); + if (flussHandle.isPartitioned()) { + props.put(ScanNodePropertyKeys.PATH_PARTITION_KEYS, + String.join(",", flussHandle.getPartitionKeys())); + } + props.put(PROP_DB_NAME, flussHandle.getDatabaseName()); + props.put(PROP_TABLE_NAME, flussHandle.getTableName()); + FlussConnectorProperties.toFlussClientConfig(catalogProperties) + .forEach((key, value) -> props.put(PROP_CLIENT_PREFIX + key, value)); + return props; + } + + @Override + public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { + Map beProperties = new LinkedHashMap<>(); + for (Map.Entry entry : nodeProperties.entrySet()) { + // Prefix-gated rather than copied wholesale: the map also carries the engine's own keys and + // the synthetic ones it injects for EXPLAIN, none of which mean anything to the scanner. + if (entry.getKey().startsWith(BE_PROPERTY_PREFIX)) { + beProperties.put(entry.getKey(), entry.getValue()); + } + } + params.setFlussProperties(beProperties); + } + + /** + * The line a regression test reads to tell which way a scan was actually planned. {@code auto} + * silently falls back to a fluss-only read, so "did this query read the lake?" is otherwise + * invisible in the plan and a union-read test would pass without having tested anything. + */ + @Override + public void appendExplainInfo(StringBuilder output, String prefix, Map nodeProperties) { + output.append(prefix) + .append("flussScan: unionRead=").append(plannedUnionRead ? "yes" : "no") + .append(", lakeSplits=0") + .append(", logRanges=").append(plannedLogRanges) + .append(", mode=") + .append(FlussConnectorProperties.unionReadMode(catalogProperties).propertyValue()) + .append("\n"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java new file mode 100644 index 00000000000000..5d74e610c72540 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java @@ -0,0 +1,279 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Split planning against a real fluss cluster started in this JVM. + * + *

{@link FlussSplitPlanTest} proves the planning logic over recorded answers; this proves the + * premise those recordings encode — that a cluster really reports a bucket nobody wrote to as offset + * 0, that the stopping offset planning takes is the one a reader would see, and that a partition's + * buckets are addressed the way this connector addresses them. A recorded answer that has drifted from + * the real one makes every unit test above pass while nothing works. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussLogScanPlanClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + private static final int BUCKETS = 3; + + /** The cluster extension drops every non-built-in database after each test, so fixtures are per-test. */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() throws Exception { + Configuration clientConf = FLUSS_CLUSTER.getClientConfig(); + connection = ConnectionFactory.createConnection(clientConf); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_scan_plan_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "log_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .build()) + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + + admin.createTable(TablePath.of(db, "log_part"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + admin.createPartition(TablePath.of(db, "log_part"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(TablePath.of(db, "log_part"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + /** + * The stopping offset a range carries has to be the offset the log had actually reached, asked of + * the same cluster the reader will read. Anything else silently truncates the scan or runs past the + * end. + */ + @Test + public void rangesStopAtTheOffsetsTheClusterReports() throws Exception { + TablePath tablePath = TablePath.of(db, "log_table"); + appendRows(tablePath, 20); + + Map latest = admin.listOffsets(tablePath, allBuckets(), new OffsetSpec.LatestSpec()) + .all().get(); + List ranges = plan("log_table"); + + Map planned = new HashMap<>(); + for (ConnectorScanRange range : ranges) { + Map props = range.getProperties(); + Assertions.assertEquals("LOG", props.get("fluss.range_type")); + // The earliest sentinel goes out verbatim; only fluss resolves it. + Assertions.assertEquals("-2", props.get("fluss.log_start_offset")); + planned.put(Integer.parseInt(props.get("fluss.bucket_id")), + Long.parseLong(props.get("fluss.log_stop_offset"))); + } + + // Every non-empty bucket, and only those, with exactly the cluster's offset. + Map expected = new HashMap<>(); + latest.forEach((bucket, offset) -> { + if (offset > 0) { + expected.put(bucket, offset); + } + }); + Assertions.assertEquals(expected, planned); + Assertions.assertFalse(expected.isEmpty(), "the fixture wrote no rows anywhere"); + long total = 0; + for (long offset : expected.values()) { + total += offset; + } + Assertions.assertEquals(20, total, "planned ranges must cover every appended row"); + } + + /** The premise behind skipping empty buckets: a cluster reports a never-written bucket as 0. */ + @Test + public void tableNothingWasWrittenToPlansNoRanges() { + Assertions.assertTrue(plan("log_table").isEmpty()); + } + + /** + * A partitioned table is planned partition by partition, addressed by fluss's own partition name + * and reported to Doris under the Hive-style one. Getting either wrong is invisible until a query + * returns the wrong partition's rows. + */ + @Test + public void partitionedRangesCarryTheClusterAssignedPartitionIdentity() throws Exception { + TablePath tablePath = TablePath.of(db, "log_part"); + appendPartitionedRows(tablePath, "2026_08_02", 6); + + List ranges = plan("log_part"); + + Assertions.assertFalse(ranges.isEmpty()); + long partitionId = admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals("2026_08_02")) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + long rows = 0; + for (ConnectorScanRange range : ranges) { + Map props = range.getProperties(); + // Only the written partition produced ranges; the empty one has nothing to read. + Assertions.assertEquals("dt=2026_08_02", props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + Assertions.assertEquals(Collections.singletonMap("dt", "2026_08_02"), + range.getPartitionValues()); + rows += Long.parseLong(props.get("fluss.log_stop_offset")); + } + Assertions.assertEquals(6, rows); + } + + /** Reading a primary-key table is refused loudly rather than answered with the log alone. */ + @Test + public void primaryKeyTablesAreStillRefusedAgainstARealCluster() throws Exception { + admin.createTable(TablePath.of(db, "pk_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(2, "id") + .build(), + true).get(); + + Assertions.assertThrows(RuntimeException.class, () -> plan("pk_table")); + } + + private List plan(String tableName) { + FlussTestSession session = new FlussTestSession(1L, "cluster-plan"); + ConnectorTableHandle handle = connector.getMetadata(session) + .getTableHandle(session, db, tableName).orElseThrow(AssertionError::new); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + return provider.planScan(session, + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + + private static void appendRows(TablePath tablePath, int rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (int i = 0; i < rows; i++) { + writer.append(GenericRow.of((long) i, BinaryString.fromString("name-" + i))); + } + writer.flush(); + } + } + + private static void appendPartitionedRows(TablePath tablePath, String partition, int rows) + throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (int i = 0; i < rows; i++) { + writer.append(GenericRow.of((long) i, BinaryString.fromString(partition))); + } + writer.flush(); + } + } + + private static List allBuckets() { + List buckets = new ArrayList<>(BUCKETS); + for (int bucket = 0; bucket < BUCKETS; bucket++) { + buckets.add(bucket); + } + return buckets; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java new file mode 100644 index 00000000000000..f400ac30b32faa --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -0,0 +1,442 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TFileScanRangeParams; + +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Split planning driven entirely off recorded admin answers, which is the point: the states worth + * asserting — a bucket nobody ever wrote to, a partition the engine pruned away, a lake table whose + * tiering has not committed once — are a nuisance to produce on a live cluster and trivial here. The + * cluster test alongside this one covers that the recorded answers match what a real cluster gives. + */ +public class FlussSplitPlanTest { + + private static final TablePath LOG_TABLE = TablePath.of("db", "log_tbl"); + + private RecordingFlussAdminOps adminOps; + private ConnectorSession session; + + @BeforeEach + public void setUp() { + adminOps = new RecordingFlussAdminOps(); + session = new FlussTestSession(1L, "q1"); + } + + // ---------------------------------------------------------------- unpartitioned log table + + @Test + public void everyNonEmptyBucketGetsARangeFromEarliestToTheOffsetPlanningSaw() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 10L, 25L, 7L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertLogRange(ranges.get(0), 0, -2L, 10L); + assertLogRange(ranges.get(1), 1, -2L, 25L); + assertLogRange(ranges.get(2), 2, -2L, 7L); + } + + /** + * All of a partition's buckets stop at one view of the table. Taking the offsets per bucket instead + * would let a bucket planned later include rows written after the query started. + */ + @Test + public void offsetsAreTakenOncePerPartition() { + registerLogTable(LOG_TABLE, 4); + latestOffsets(null, 1L, 1L, 1L, 1L); + + plan(LOG_TABLE, catalog()); + + long offsetCalls = adminOps.calls.stream().filter(c -> c.startsWith("listOffsets(")).count(); + Assertions.assertEquals(1, offsetCalls, adminOps.calls.toString()); + Assertions.assertTrue(adminOps.calls.get(0).contains("[0, 1, 2, 3]"), adminOps.calls.toString()); + Assertions.assertTrue(adminOps.calls.get(0).contains("LatestSpec"), adminOps.calls.toString()); + } + + /** A bucket nobody has written to yields no range: an empty scanner would only cost a round trip. */ + @Test + public void neverWrittenBucketsAreSkipped() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 0L, 12L, 0L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(1, ranges.size()); + assertLogRange(ranges.get(0), 1, -2L, 12L); + } + + @Test + public void anEmptyTablePlansNothing() { + registerLogTable(LOG_TABLE, 2); + latestOffsets(null, 0L, 0L); + + Assertions.assertTrue(plan(LOG_TABLE, catalog()).isEmpty()); + } + + // ---------------------------------------------------------------- partitioned log table + + @Test + public void eachPartitionContributesItsOwnBuckets() { + registerPartitionedLogTable(2, "20260101", "20260102"); + latestOffsets("20260101", 5L, 6L); + latestOffsets("20260102", 7L, 0L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertPartition(ranges.get(0), "dt=20260101", 100L, 0, 5L); + assertPartition(ranges.get(1), "dt=20260101", 100L, 1, 6L); + assertPartition(ranges.get(2), "dt=20260102", 101L, 0, 7L); + } + + @Test + public void onlyThePartitionsTheEnginePrunedToAreScanned() { + registerPartitionedLogTable(1, "20260101", "20260102", "20260103"); + latestOffsets("20260102", 9L); + + List ranges = plan(LOG_TABLE, catalog(), + Collections.singletonList("dt=20260102")); + + Assertions.assertEquals(1, ranges.size()); + assertPartition(ranges.get(0), "dt=20260102", 101L, 0, 9L); + // The other two partitions must not even be asked for their offsets. + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("listOffsets(")).count(), + adminOps.calls.toString()); + } + + /** + * The pruned names the engine hands back are the ones the metadata listing produced, so planning has + * to render partition names exactly the same way. Rendering them separately is how "prune to one + * partition, then match nothing and scan zero rows" happens, and it looks like an empty table. + */ + @Test + public void plannedPartitionNamesMatchTheOnesPruningSaw() { + registerPartitionedLogTable(1, "20260101", "20260102"); + latestOffsets("20260101", 3L); + latestOffsets("20260102", 3L); + ConnectorTableHandle handle = handle(LOG_TABLE); + + List listedNames = new ArrayList<>(); + for (ConnectorPartitionInfo partition + : metadata().listPartitions(session, handle, Optional.empty())) { + listedNames.add(partition.getPartitionName()); + } + // Feeding the listing's own names back as the pruned set must select every partition. + List ranges = plan(LOG_TABLE, catalog(), listedNames); + + Assertions.assertEquals(Arrays.asList("dt=20260101", "dt=20260102"), listedNames); + Assertions.assertEquals(2, ranges.size()); + } + + /** Dropped between pruning and planning: nothing left to read, and nothing to fail about. */ + @Test + public void prunedPartitionThatNoLongerExistsIsSkipped() { + registerPartitionedLogTable(1, "20260101"); + latestOffsets("20260101", 3L); + + List ranges = plan(LOG_TABLE, catalog(), + Arrays.asList("dt=20260101", "dt=19990101")); + + Assertions.assertEquals(1, ranges.size()); + assertPartition(ranges.get(0), "dt=20260101", 100L, 0, 3L); + } + + @Test + public void partitionColumnsAreDeclaredToTheEngineNotReadBackFromTheScanner() { + registerPartitionedLogTable(1, "20260101"); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertEquals("dt", props.get(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + } + + @Test + public void anUnpartitionedTableDeclaresNoPartitionKeys() { + registerLogTable(LOG_TABLE, 1); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertFalse(props.containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + } + + // ---------------------------------------------------------------- what is refused, and why + + @Test + public void primaryKeyTableIsRefusedRatherThanReadAsALog() { + TablePath pkTable = TablePath.of("db", "pk_tbl"); + adminOps.tableInfos.put(pkTable, FlussTestTables.builder(pkTable) + .column("id", DataTypes.INT().copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(2, "id") + .build()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(pkTable, catalog())); + Assertions.assertTrue(e.getMessage().contains("primary-key"), e.getMessage()); + } + + /** + * A lake table read as fluss-only returns only what the log still holds, dropping everything tiering + * has moved into the lake — a query that succeeds with missing rows. Until the union read exists, + * the refusal is the correct answer. + */ + @Test + public void tieredLakeTableIsRefusedUntilTheUnionReadExists() { + registerLakeTable(2); + adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(LOG_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("not supported yet"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains(FlussConnectorProperties.UNION_READ_MODE), + e.getMessage()); + } + + /** Nothing in the lake means the log holds the whole table, so the fluss-only read IS complete. */ + @Test + public void lakeTableThatHasNeverTieredFallsBackToTheLogAlone() { + registerLakeTable(2); + latestOffsets(null, 4L, 4L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + } + + @Test + public void requiredModeRefusesTheFallbackSoATestCannotPassByAccident() { + registerLakeTable(2); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(LOG_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("no readable lake snapshot"), e.getMessage()); + } + + /** Disabled is the user asking for the fluss-only read on purpose; the lake is not even consulted. */ + @Test + public void disabledModeReadsTheLogWithoutAskingAboutTheLake() { + registerLakeTable(2); + latestOffsets(null, 4L, 4L); + + List ranges = + plan(LOG_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled")); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + // ---------------------------------------------------------------- what BE and EXPLAIN receive + + @Test + public void scanLevelParamsCarryTheClientConfigAndTableIdentity() { + registerLogTable(LOG_TABLE, 1); + Map catalog = catalog(); + catalog.put("fluss.client.writer.batch-size", "2mb"); + + TFileScanRangeParams params = new TFileScanRangeParams(); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog); + provider.populateScanLevelParams(params, nodeProperties(LOG_TABLE, catalog)); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.db_name", "db"); + expected.put("fluss.table_name", "log_tbl"); + expected.put("fluss.client.bootstrap.servers", "localhost:9123"); + expected.put("fluss.client.client.writer.batch-size", "2mb"); + Assertions.assertEquals(expected, params.getFlussProperties()); + } + + /** The engine's own keys are not the scanner's; forwarding them wholesale would be noise at best. */ + @Test + public void scanLevelParamsDropTheEngineOnlyKeys() { + registerPartitionedLogTable(1, "20260101"); + Map nodeProps = new LinkedHashMap<>(nodeProperties(LOG_TABLE, catalog())); + nodeProps.put(ScanNodePropertyKeys.SYNTHETIC_TOTAL_READ_SPLITS, "3"); + + TFileScanRangeParams params = new TFileScanRangeParams(); + new FlussScanPlanProvider(adminOps, catalog()).populateScanLevelParams(params, nodeProps); + + Assertions.assertFalse(params.getFlussProperties() + .containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + Assertions.assertFalse(params.getFlussProperties() + .containsKey(ScanNodePropertyKeys.SYNTHETIC_TOTAL_READ_SPLITS)); + } + + /** + * {@code auto} falls back to a fluss-only read without saying so anywhere else in the plan, so a + * union-read regression test has nothing to assert on but this line. + */ + @Test + public void explainReportsHowTheScanWasActuallyPlanned() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 1L, 0L, 5L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog()); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, " ", Collections.emptyMap()); + + Assertions.assertEquals( + " flussScan: unionRead=no, lakeSplits=0, logRanges=2, mode=auto\n", output.toString()); + } + + @Test + public void explainReportsTheConfiguredMode() { + registerLogTable(LOG_TABLE, 1); + latestOffsets(null, 1L); + Map catalog = catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertTrue(output.toString().contains("mode=disabled"), output.toString()); + } + + // ---------------------------------------------------------------- helpers + + private Map catalog() { + Map properties = new HashMap<>(); + properties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123"); + return properties; + } + + private Map catalog(String key, String value) { + Map properties = catalog(); + properties.put(key, value); + return properties; + } + + private FlussConnectorMetadata metadata() { + return new FlussConnectorMetadata(adminOps, new FlussTypeMapping.Options(false, false)); + } + + private ConnectorTableHandle handle(TablePath tablePath) { + return FlussTableHandle.of(adminOps.tableInfos.get(tablePath)); + } + + private List plan(TablePath tablePath, Map catalogProperties) { + return plan(tablePath, catalogProperties, Collections.emptyList()); + } + + private List plan(TablePath tablePath, Map catalogProperties, + List requiredPartitions) { + return new FlussScanPlanProvider(adminOps, catalogProperties) + .planScan(session, request(handle(tablePath), requiredPartitions)); + } + + private Map nodeProperties(TablePath tablePath, Map catalogProperties) { + return new FlussScanPlanProvider(adminOps, catalogProperties).getScanNodeProperties( + session, handle(tablePath), Collections.emptyList(), Optional.empty()); + } + + private static ConnectorScanRequest request(ConnectorTableHandle handle, List requiredPartitions) { + return ConnectorScanRequest.builder(handle, Collections.emptyList()) + .requiredPartitions(requiredPartitions) + .build(); + } + + private void registerLogTable(TablePath tablePath, int buckets) { + adminOps.tableInfos.put(tablePath, FlussTestTables.builder(tablePath) + .column("id", DataTypes.INT()) + .column("v", DataTypes.STRING()) + .buckets(buckets) + .build()); + } + + /** A log table partitioned by {@code dt}, with partition ids 100, 101, ... in the order given. */ + private void registerPartitionedLogTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionedBy("dt") + .buckets(buckets) + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(LOG_TABLE, partitions); + } + + private void registerLakeTable(int buckets) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .buckets(buckets) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .build()); + } + + /** Latest offsets for buckets 0..n-1 of {@code partitionName} ({@code null} = unpartitioned). */ + private void latestOffsets(String partitionName, long... offsets) { + Map byBucket = new LinkedHashMap<>(); + for (int bucket = 0; bucket < offsets.length; bucket++) { + byBucket.put(bucket, offsets[bucket]); + } + adminOps.latestOffsetsByPartition.put(partitionName, byBucket); + } + + private static void assertLogRange(ConnectorScanRange range, int bucket, long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("LOG", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + + private static void assertPartition(ConnectorScanRange range, String partitionName, + long partitionId, int bucket, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals(partitionName, props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + assertLogRange(range, bucket, -2L, stop); + } + +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java index fb47ab4ccf4f4c..a6533935b0cb05 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -20,6 +20,7 @@ import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.PartitionSpec; @@ -56,6 +57,13 @@ class RecordingFlussAdminOps implements FlussAdminOps { final Map tableInfos = new HashMap<>(); final Map> partitionsByTable = new HashMap<>(); final Map statsByTable = new HashMap<>(); + /** + * Latest log offset per bucket, keyed by fluss's own partition name ({@code null} for an + * unpartitioned table) — what split planning stops each bucket at. + */ + final Map> latestOffsetsByPartition = new HashMap<>(); + /** The readable lake snapshot, or {@code null} to answer as a table nothing has been tiered from. */ + LakeSnapshot readableLakeSnapshot; /** When set, every call throws this instead of answering — the "cluster is unreachable" case. */ RuntimeException failure; @@ -140,18 +148,36 @@ public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionNam @Override public LakeSnapshot getReadableLakeSnapshot(TablePath tablePath) { - throw notProgrammed("getReadableLakeSnapshot"); + calls.add("getReadableLakeSnapshot(" + tablePath + ")"); + if (readableLakeSnapshot == null) { + // What a real cluster answers for a table tiering has never committed for; the connector + // discriminates on this exact type to decide whether a union read is possible at all. + throw new LakeTableSnapshotNotExistException( + "Lake snapshot for table '" + tablePath + "' does not exist."); + } + return readableLakeSnapshot; } @Override public Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec) { - throw notProgrammed("listOffsets"); + return recordedOffsets(tablePath, null, buckets, offsetSpec); } @Override public Map listOffsets(TablePath tablePath, String partitionName, Collection buckets, OffsetSpec offsetSpec) { - throw notProgrammed("listOffsets"); + return recordedOffsets(tablePath, partitionName, buckets, offsetSpec); + } + + private Map recordedOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + calls.add("listOffsets(" + tablePath + (partitionName == null ? "" : ", " + partitionName) + + ", " + buckets + ", " + offsetSpec.getClass().getSimpleName() + ")"); + Map offsets = latestOffsetsByPartition.get(partitionName); + if (offsets == null) { + throw new IllegalStateException("no offsets programmed for partition '" + partitionName + "'"); + } + return offsets; } private static UnsupportedOperationException notProgrammed(String method) { From 68682775ed5de0858952385f89feca319f600ef3 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 21:58:32 +0800 Subject: [PATCH 07/35] [feat](fluss) Read a fluss log range in the BE java extension Turns one scan range -- one bucket of one partition over a bounded offset range -- into rows in Doris's vector table, by driving a fluss log scanner and converting each fluss row column by column. Where the read stops needs all three of fluss's own conditions, taken from its bounded reader in KvSnapshotAndLogBatchScanner: drop a record at or past the stopping offset, stop right after the record before it, and stop when the fetch has consumed up to the stopping offset without yielding a record there. The second exists because the record AT the stopping offset may never be written -- that offset is where the log had got to, not a row -- so polling on would block forever. The third covers a tail of control records, which take offsets but are never handed to a scanner. Byte-valued columns dispatch on the FLUSS type, not the Doris one. The two do not line up: fluss BYTES maps to Doris STRING unless the catalog opts into VARBINARY, so one column may be asked for either way, and CHAR and BINARY are stored fixed-width and read back as nothing without their declared length. fluss-client already carries a relocated Arrow, allocator included, so nothing here declares one; the arrow coordinates its pom marks provided are not the ones it uses. What it does need is the java.nio add-opens, which bin/start_be.sh already passes. Tests run against a real cluster because both ends of the conversion are someone else's format. They read values back through the STRING mapping rather than VARBINARY: java-common's getBytesVarbinary decodes a StringView layout that neither appendVarbinary nor BE's _fill_varbinary_column writes, so that read-back path cannot check a byte value. Production is unaffected -- the write side and BE agree. Co-Authored-By: Claude Opus 5 (1M context) --- build.sh | 2 + fe/be-java-extensions/fluss-scanner/pom.xml | 153 +++++ .../apache/doris/fluss/FlussColumnValue.java | 334 +++++++++++ .../apache/doris/fluss/FlussJniScanner.java | 337 +++++++++++ .../src/main/resources/package.xml | 47 ++ .../doris/fluss/FlussJniScannerLogTest.java | 558 ++++++++++++++++++ fe/be-java-extensions/pom.xml | 1 + fe/fe-connector/fe-connector-fluss/pom.xml | 10 +- 8 files changed, 1438 insertions(+), 4 deletions(-) create mode 100644 fe/be-java-extensions/fluss-scanner/pom.xml create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java diff --git a/build.sh b/build.sh index 39e27e91c8badc..d76c443ceb4245 100755 --- a/build.sh +++ b/build.sh @@ -746,6 +746,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/java-udf") modules+=("be-java-extensions/jdbc-scanner") modules+=("be-java-extensions/paimon-scanner") + modules+=("be-java-extensions/fluss-scanner") modules+=("be-java-extensions/trino-connector-scanner") modules+=("be-java-extensions/max-compute-connector") # lakesoul-scanner has been deprecated @@ -1196,6 +1197,7 @@ EOF extensions_modules+=("jdbc-scanner") extensions_modules+=("hadoop-hudi-scanner") extensions_modules+=("paimon-scanner") + extensions_modules+=("fluss-scanner") extensions_modules+=("trino-connector-scanner") extensions_modules+=("max-compute-connector") # lakesoul-scanner has been deprecated diff --git a/fe/be-java-extensions/fluss-scanner/pom.xml b/fe/be-java-extensions/fluss-scanner/pom.xml new file mode 100644 index 00000000000000..7f771902d39b36 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/pom.xml @@ -0,0 +1,153 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + fluss-scanner + + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + provided + + + + + org.apache.fluss + fluss-client + ${fluss.version} + + + + org.junit.jupiter + junit-jupiter + test + + + + + org.apache.fluss + fluss-server + ${fluss.version} + test + + + + org.apache.fluss + fluss-server + ${fluss.version} + test-jar + test + + + + org.apache.fluss + fluss-test-utils + ${fluss.version} + test + + + + org.apache.curator + curator-test + 5.4.0 + test + + + + + fluss-scanner + ${project.basedir}/target/ + + + org.apache.maven.plugins + maven-surefire-plugin + + + --add-opens=java.base/java.nio=ALL-UNNAMED + + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java new file mode 100644 index 00000000000000..98df50d05248b3 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.fluss.row.DataGetters; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * One column of one fluss row, read the way Doris's vector table wants it. + * + *

Two fluss-specific things this has to get right, both of which a paimon-shaped port would miss: + * + *

+ * + *

Instances are reused across rows and, for nested values, across containers: the vector table + * consumes an unpacked value before the parent moves on, so a cache keyed by position is safe and + * keeps a wide nested column from allocating per row. + */ +public class FlussColumnValue implements ColumnValue { + + private static final Map DORIS_TIME_ZONE_ALIASES; + + static { + Map aliases = new HashMap<>(ZoneId.SHORT_IDS); + // The scanner cannot depend on FE's TimeUtils, so keep its accepted aliases and CST + // interpretation identical at this JNI boundary (same list as the paimon scanner's). + aliases.put("CST", "Asia/Shanghai"); + aliases.put("PRC", "Asia/Shanghai"); + aliases.put("UTC", "UTC"); + aliases.put("GMT", "UTC"); + DORIS_TIME_ZONE_ALIASES = Collections.unmodifiableMap(aliases); + } + + private DataGetters record; + private int idx; + private ColumnType dorisType; + private DataType flussType; + private ZoneId timeZone; + + // Lazy: a scalar column never pays for the nested-reuse bookkeeping. + private List arrayValues; + private List mapKeys; + private List mapValues; + private List structValues; + + public FlussColumnValue(String timeZone) { + this.timeZone = resolveTimeZone(timeZone); + } + + private FlussColumnValue(DataGetters record, int idx, ColumnType dorisType, DataType flussType, + ZoneId timeZone) { + this.record = record; + this.idx = idx; + this.dorisType = dorisType; + this.flussType = flussType; + this.timeZone = timeZone; + } + + /** Points this value at the row the scanner is currently emitting. */ + public void setRow(InternalRow record) { + this.record = record; + } + + /** Points this value at one column of that row. */ + public void setIdx(int idx, ColumnType dorisType, DataType flussType) { + this.idx = idx; + this.dorisType = dorisType; + this.flussType = flussType; + } + + @Override + public boolean canGetStringAsBytes() { + return true; + } + + @Override + public boolean isNull() { + boolean isNull = record.isNullAt(idx); + if (isNull) { + // A null container has no live descendants; drop wrappers its previous row left behind. + clearChildCaches(); + } + return isNull; + } + + @Override + public boolean getBoolean() { + return record.getBoolean(idx); + } + + @Override + public byte getByte() { + return record.getByte(idx); + } + + @Override + public short getShort() { + return record.getShort(idx); + } + + @Override + public int getInt() { + return record.getInt(idx); + } + + @Override + public long getLong() { + return record.getLong(idx); + } + + @Override + public float getFloat() { + return record.getFloat(idx); + } + + @Override + public double getDouble() { + return record.getDouble(idx); + } + + @Override + public BigInteger getBigInteger() { + return BigInteger.valueOf(record.getInt(idx)); + } + + @Override + public BigDecimal getDecimal() { + return record.getDecimal(idx, dorisType.getPrecision(), dorisType.getScale()).toBigDecimal(); + } + + @Override + public String getString() { + return new String(readBytes(), StandardCharsets.UTF_8); + } + + @Override + public byte[] getStringAsBytes() { + return readBytes(); + } + + @Override + public LocalDate getDate() { + return LocalDate.ofEpochDay(record.getInt(idx)); + } + + @Override + public LocalDateTime getDateTime() { + if (flussType instanceof LocalZonedTimestampType) { + // Stored as an instant, so it only becomes a wall clock in a zone — the session's. + return LocalDateTime.ofInstant( + record.getTimestampLtz(idx, precisionOf(flussType)).toInstant(), timeZone); + } + return record.getTimestampNtz(idx, precisionOf(flussType)).toLocalDateTime(); + } + + @Override + public LocalDateTime getTimeStampTz() { + // TIMESTAMPTZ keeps the instant itself, and Doris carries it as the UTC wall clock — NOT the + // session zone's, which is what getDateTime above applies. + return LocalDateTime.ofInstant( + record.getTimestampLtz(idx, precisionOf(flussType)).toInstant(), ZoneOffset.UTC); + } + + @Override + public byte[] getBytes() { + return readBytes(); + } + + @Override + public void unpackArray(List values) { + InternalArray array = record.getArray(idx); + if (arrayValues == null) { + arrayValues = new ArrayList<>(); + } + ColumnType elementDorisType = dorisType.getChildTypes().get(0); + DataType elementFlussType = ((ArrayType) flussType).getElementType(); + for (int i = 0; i < array.size(); i++) { + values.add(reuse(arrayValues, i, array, i, elementDorisType, elementFlussType)); + } + trim(arrayValues, array.size()); + } + + @Override + public void unpackMap(List keys, List values) { + InternalMap map = record.getMap(idx); + if (mapKeys == null) { + mapKeys = new ArrayList<>(); + mapValues = new ArrayList<>(); + } + InternalArray keyArray = map.keyArray(); + ColumnType keyDorisType = dorisType.getChildTypes().get(0); + DataType keyFlussType = ((MapType) flussType).getKeyType(); + for (int i = 0; i < keyArray.size(); i++) { + keys.add(reuse(mapKeys, i, keyArray, i, keyDorisType, keyFlussType)); + } + trim(mapKeys, keyArray.size()); + + InternalArray valueArray = map.valueArray(); + ColumnType valueDorisType = dorisType.getChildTypes().get(1); + DataType valueFlussType = ((MapType) flussType).getValueType(); + for (int i = 0; i < valueArray.size(); i++) { + values.add(reuse(mapValues, i, valueArray, i, valueDorisType, valueFlussType)); + } + trim(mapValues, valueArray.size()); + } + + @Override + public void unpackStruct(List structFieldIndex, List values) { + RowType rowType = (RowType) flussType; + // The indexes are into the FULL child list, so the nested row must keep its whole arity. + InternalRow row = record.getRow(idx, rowType.getFieldCount()); + if (structValues == null) { + structValues = new ArrayList<>(); + } + for (int i : structFieldIndex) { + values.add(reuse(structValues, i, row, i, dorisType.getChildTypes().get(i), + rowType.getTypeAt(i))); + } + } + + /** + * The raw bytes of a character or binary column, chosen by the FLUSS type rather than the Doris + * one. That distinction matters because the two do not line up one to one: fluss BYTES maps to + * Doris STRING unless the catalog turns on VARBINARY mapping, so the same fluss column can be + * asked for as a string here or as bytes there. Which accessor fluss needs is a property of how + * fluss stored the value, and CHAR and BINARY are stored fixed-width — reading them without their + * declared length gets nothing back. + */ + private byte[] readBytes() { + switch (flussType.getTypeRoot()) { + case CHAR: + return record.getChar(idx, ((CharType) flussType).getLength()).toBytes(); + case STRING: + return record.getString(idx).toBytes(); + case BINARY: + return record.getBinary(idx, ((BinaryType) flussType).getLength()); + case BYTES: + return record.getBytes(idx); + default: + throw new IllegalStateException( + "fluss type " + flussType + " has no byte representation to read"); + } + } + + private FlussColumnValue reuse(List cache, int cacheIndex, + DataGetters childRecord, int childIndex, ColumnType childDorisType, DataType childFlussType) { + while (cache.size() <= cacheIndex) { + cache.add(null); + } + FlussColumnValue value = cache.get(cacheIndex); + if (value == null) { + value = new FlussColumnValue(childRecord, childIndex, childDorisType, childFlussType, timeZone); + cache.set(cacheIndex, value); + } else { + value.record = childRecord; + value.idx = childIndex; + value.dorisType = childDorisType; + value.flussType = childFlussType; + value.timeZone = timeZone; + } + return value; + } + + private static int precisionOf(DataType type) { + return org.apache.fluss.types.DataTypeChecks.getPrecision(type); + } + + private static ZoneId resolveTimeZone(String timeZone) { + return ZoneId.of(timeZone, DORIS_TIME_ZONE_ALIASES); + } + + private static void trim(List cache, int liveSize) { + if (cache.size() > liveSize) { + // Keep only what the CURRENT container can address, not its historical maximum. + cache.subList(liveSize, cache.size()).clear(); + } + } + + private void clearChildCaches() { + arrayValues = null; + mapKeys = null; + mapValues = null; + structValues = null; + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java new file mode 100644 index 00000000000000..b05150bebe0762 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java @@ -0,0 +1,337 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnType; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +/** + * Reads one fluss scan range — one bucket of one partition, over a bounded log offset range. + * + *

The parameters are the two maps FE built, merged by BE: the scan-level one (connection, table) + * and the per-range one (which bucket, which offsets). Nothing between FE and here type-checks them, + * so every one this class needs is read through {@link #required}, which names the missing key rather + * than letting a null reach fluss. + * + *

Where the read stops. A fluss log scanner is a streaming reader with no end; the range's + * stopping offset is what bounds it, and reaching it has to be detected three ways, all of which come + * from fluss's own bounded reader ({@code KvSnapshotAndLogBatchScanner#pollLogRecords}): + *

+ * + *

Partition columns are not read here. FE declares them to the engine, which leaves them out of + * {@code required_fields} and fills them from the range itself, so the projection this builds covers + * only the data columns — the same division the paimon scanner works under, and the one a union read + * needs both halves to agree on. + */ +public class FlussJniScanner extends JniScanner { + + private static final Logger LOG = LoggerFactory.getLogger(FlussJniScanner.class); + + /** Prefix for the fluss client configuration; the rest of the key is fluss's own option name. */ + private static final String CLIENT_PREFIX = "fluss.client."; + private static final String DB_NAME = "fluss.db_name"; + private static final String TABLE_NAME = "fluss.table_name"; + private static final String RANGE_TYPE = "fluss.range_type"; + private static final String PARTITION_ID = "fluss.partition_id"; + private static final String BUCKET_ID = "fluss.bucket_id"; + private static final String LOG_START_OFFSET = "fluss.log_start_offset"; + private static final String LOG_STOP_OFFSET = "fluss.log_stop_offset"; + + private static final String RANGE_TYPE_LOG = "LOG"; + + /** + * How long one poll waits for data. Only affects how often the loop spins, never correctness: the + * loop keeps polling until one of the three stop conditions above fires. + */ + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(1); + + private final Map params; + private final ClassLoader classLoader; + private final FlussColumnValue columnValue; + + private final long logStopOffset; + private final long logStartOffset; + private final int bucketId; + /** {@code null} on an unpartitioned table, which fluss subscribes to by bucket alone. */ + private final Long partitionId; + + private Connection connection; + private Table table; + private LogScanner logScanner; + private TableBucket tableBucket; + + /** Fluss types of the projected columns, positionally aligned with {@link #fields}. */ + private List projectedTypes; + + private Iterator pending = Collections.emptyIterator(); + private boolean finished; + private long rowsRead; + + public FlussJniScanner(int batchSize, Map params) { + this.params = params; + this.classLoader = this.getClass().getClassLoader(); + + String rangeType = required(RANGE_TYPE); + if (!RANGE_TYPE_LOG.equals(rangeType)) { + // Primary-key and union ranges are planned but not read yet; failing here beats reading + // a primary-key table as a raw changelog and returning superseded rows. + throw new IllegalArgumentException( + "fluss scan range type '" + rangeType + "' is not supported yet; expected " + + RANGE_TYPE_LOG); + } + // Every range field is parsed here, before open() creates a connection: a range that cannot be + // read should say so without having contacted the cluster first. + this.logStopOffset = Long.parseLong(required(LOG_STOP_OFFSET)); + this.logStartOffset = Long.parseLong(required(LOG_START_OFFSET)); + this.bucketId = Integer.parseInt(required(BUCKET_ID)); + String partition = params.get(PARTITION_ID); + this.partitionId = partition == null ? null : Long.parseLong(partition); + + String[] requiredFields = splitOn(params.get("required_fields"), ","); + String[] requiredTypes = splitOn(params.get("columns_types"), "#"); + if (requiredFields.length != requiredTypes.length) { + throw new IllegalArgumentException("required_fields size " + requiredFields.length + + " does not match columns_types size " + requiredTypes.length); + } + ColumnType[] columnTypes = new ColumnType[requiredTypes.length]; + for (int i = 0; i < requiredTypes.length; i++) { + columnTypes[i] = ColumnType.parseType(requiredFields[i], requiredTypes[i]); + } + initTableInfo(columnTypes, requiredFields, batchSize); + this.columnValue = new FlussColumnValue( + params.getOrDefault("time_zone", TimeZone.getDefault().getID())); + } + + @Override + public void open() throws IOException { + ClassLoader callerLoader = Thread.currentThread().getContextClassLoader(); + // The fluss client spawns its own threads (netty IO, metadata updater) while connecting, and a + // thread inherits the context classloader of whoever created it. Started under BE's loader they + // would not see fluss at all. + Thread.currentThread().setContextClassLoader(classLoader); + try { + connection = ConnectionFactory.createConnection(clientConfig()); + table = connection.getTable(TablePath.of(required(DB_NAME), required(TABLE_NAME))); + + RowType rowType = table.getTableInfo().getRowType(); + int[] projection = projectionOf(rowType); + projectedTypes = new ArrayList<>(projection.length); + for (int index : projection) { + projectedTypes.add(rowType.getTypeAt(index)); + } + + logScanner = table.newScan().project(projection).createLogScanner(); + subscribe(); + } catch (Throwable e) { + try { + close(); + } catch (IOException closeFailure) { + e.addSuppressed(closeFailure); + } + throw new IOException("Failed to open the fluss scanner for " + + params.get(DB_NAME) + "." + params.get(TABLE_NAME) + + " bucket " + params.get(BUCKET_ID), e); + } finally { + Thread.currentThread().setContextClassLoader(callerLoader); + } + } + + private void subscribe() { + long tableId = table.getTableInfo().getTableId(); + if (partitionId == null) { + tableBucket = new TableBucket(tableId, bucketId); + logScanner.subscribe(bucketId, logStartOffset); + } else { + tableBucket = new TableBucket(tableId, partitionId, bucketId); + logScanner.subscribe(partitionId, bucketId, logStartOffset); + } + } + + /** + * Where each required column sits in the table's row type. Resolved by NAME at open time rather + * than shipped as indexes from FE: the two are separated by planning, and a column added in + * between would shift every index after it. + */ + private int[] projectionOf(RowType rowType) { + List fieldNames = rowType.getFieldNames(); + int[] projection = new int[fields.length]; + for (int i = 0; i < fields.length; i++) { + int index = fieldNames.indexOf(fields[i]); + if (index < 0) { + throw new IllegalStateException("Column '" + fields[i] + + "' is not in the fluss table, which has " + fieldNames + + ". The table's schema changed after the query was planned."); + } + projection[i] = index; + } + return projection; + } + + private Configuration clientConfig() { + Configuration config = new Configuration(); + for (Map.Entry entry : params.entrySet()) { + if (entry.getKey().startsWith(CLIENT_PREFIX)) { + config.setString(entry.getKey().substring(CLIENT_PREFIX.length()), entry.getValue()); + } + } + return config; + } + + @Override + protected int getNext() throws IOException { + int rows = 0; + while (rows < getBatchSize()) { + if (!pending.hasNext()) { + if (finished) { + break; + } + pending = poll(); + continue; + } + ScanRecord record = pending.next(); + if (record.logOffset() >= logStopOffset) { + // Past the end of this range: another query's rows, not ours. + finished = true; + pending = Collections.emptyIterator(); + break; + } + columnValue.setRow(record.getRow()); + for (int i = 0; i < fields.length; i++) { + columnValue.setIdx(i, types[i], projectedTypes.get(i)); + appendData(i, columnValue); + } + rows++; + if (record.logOffset() >= logStopOffset - 1) { + // The last record of the range. Do not poll again: the record AT the stopping offset + // may not exist, and waiting for it never returns. + finished = true; + pending = Collections.emptyIterator(); + break; + } + } + if (fields.length == 0 && rows > 0) { + // A count-shaped read projects nothing; the vector table still needs the row count. + vectorTable.appendVirtualData(rows); + } + rowsRead += rows; + return rows; + } + + private Iterator poll() { + ScanRecords scanRecords = logScanner.poll(POLL_TIMEOUT); + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { + // The fetch reached the end of the range without necessarily yielding a record there — the + // tail can be control records, which take offsets but are never scanned. Without this the + // loop would poll for a row that is never coming. + finished = true; + } + return scanRecords.records(tableBucket).iterator(); + } + + @Override + public void close() throws IOException { + IOException failure = null; + // Close everything even if an earlier close throws: a leaked fluss connection keeps its netty + // and metadata-updater threads alive for the life of the BE process. + failure = closeQuietly(logScanner, "log scanner", failure); + logScanner = null; + failure = closeQuietly(table, "table", failure); + table = null; + failure = closeQuietly(connection, "connection", failure); + connection = null; + pending = Collections.emptyIterator(); + if (failure != null) { + throw failure; + } + } + + private IOException closeQuietly(AutoCloseable closeable, String what, IOException previous) { + if (closeable == null) { + return previous; + } + try { + closeable.close(); + return previous; + } catch (Exception e) { + LOG.warn("Failed to close the fluss {}", what, e); + IOException failure = new IOException("Failed to close the fluss " + what, e); + if (previous == null) { + return failure; + } + previous.addSuppressed(failure); + return previous; + } + } + + @Override + public Map getStatistics() { + Map statistics = new HashMap<>(); + statistics.put("counter:FlussJniRowsRead", String.valueOf(rowsRead)); + statistics.put("gauge:FlussJniRequiredFieldCount", String.valueOf(fields.length)); + return statistics; + } + + private String required(String key) { + String value = params.get(key); + if (value == null) { + throw new IllegalArgumentException( + "fluss scanner parameter '" + key + "' is missing; got keys " + params.keySet()); + } + return value; + } + + private static String[] splitOn(String value, String separator) { + if (value == null || value.isEmpty()) { + return new String[0]; + } + return value.split(separator); + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml b/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml new file mode 100644 index 00000000000000..6566bb46f110d4 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml @@ -0,0 +1,47 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + + + + metaInf-services + + + diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java new file mode 100644 index 00000000000000..eb82c5a317d4a7 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java @@ -0,0 +1,558 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The scanner against a real fluss cluster started in this JVM. + * + *

Everything here is end to end on purpose. The whole job of this class is to turn what a fluss + * cluster actually stores into what Doris's vector table expects, and both ends of that are other + * people's formats: a test that fed it hand-built rows would only check the conversion against our own + * idea of fluss's encoding. So the fixtures are written through the fluss client — which puts them in + * the real arrow log format — and read back through the scanner into a real off-heap vector table. + * + *

Named {@code ...Test} rather than {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerLogTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so a fixture's rows land in a known place and the offsets are the row numbers. */ + private static final int BUCKETS = 1; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_scanner_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- values + + /** + * Every type the connector maps, written by fluss and read back through the scanner, plus an + * all-null row. The null row is not padding: a fluss column read through the wrong accessor often + * still produces a plausible value for real data and only misbehaves on null, and a value that is + * null on the way in but not on the way out is the failure this catches. + */ + @Test + public void everyMappedTypeSurvivesTheRoundTrip() throws Exception { + TablePath tablePath = TablePath.of(db, "all_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_boolean", DataTypes.BOOLEAN()) + .column("c_tinyint", DataTypes.TINYINT()) + .column("c_smallint", DataTypes.SMALLINT()) + .column("c_int", DataTypes.INT()) + .column("c_bigint", DataTypes.BIGINT()) + .column("c_float", DataTypes.FLOAT()) + .column("c_double", DataTypes.DOUBLE()) + .column("c_char", DataTypes.CHAR(5)) + .column("c_string", DataTypes.STRING()) + .column("c_decimal", DataTypes.DECIMAL(20, 4)) + .column("c_date", DataTypes.DATE()) + .column("c_timestamp", DataTypes.TIMESTAMP(6)) + .column("c_timestamp_ltz", DataTypes.TIMESTAMP_LTZ(6)) + .column("c_binary", DataTypes.BINARY(4)) + .column("c_bytes", DataTypes.BYTES()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + + LocalDateTime timestamp = LocalDateTime.of(2026, 8, 2, 12, 34, 56, 123456000); + Instant instant = Instant.parse("2026-08-02T04:34:56.123456Z"); + appendRows(tablePath, + GenericRow.of(true, (byte) 1, (short) 2, 3, 4L, 5.5f, 6.5d, + BinaryString.fromString("abcde"), BinaryString.fromString("hello"), + Decimal.fromBigDecimal(new BigDecimal("12345.6789"), 20, 4), + (int) LocalDate.of(2026, 8, 2).toEpochDay(), + TimestampNtz.fromLocalDateTime(timestamp), + TimestampLtz.fromInstant(instant), + new byte[] {1, 2, 3, 4}, new byte[] {9, 8}), + GenericRow.of(null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null)); + + Object[][] rows = scanAll(tablePath, columns( + "c_boolean", "boolean", + "c_tinyint", "tinyint", + "c_smallint", "smallint", + "c_int", "int", + "c_bigint", "bigint", + "c_float", "float", + "c_double", "double", + "c_char", "char(5)", + "c_string", "string", + "c_decimal", "decimal(20,4)", + "c_date", "datev2", + "c_timestamp", "datetimev2(6)", + "c_timestamp_ltz", "timestamptz(6)", + "c_binary", "varbinary", + "c_bytes", "varbinary"), 0, 2, 1024); + + Assertions.assertEquals(2, rows.length); + Object[] row = rows[0]; + Assertions.assertEquals(true, row[0]); + Assertions.assertEquals((byte) 1, row[1]); + Assertions.assertEquals((short) 2, row[2]); + Assertions.assertEquals(3, row[3]); + Assertions.assertEquals(4L, row[4]); + Assertions.assertEquals(5.5f, row[5]); + Assertions.assertEquals(6.5d, row[6]); + Assertions.assertEquals("abcde", row[7]); + Assertions.assertEquals("hello", row[8]); + Assertions.assertEquals(new BigDecimal("12345.6789"), row[9]); + Assertions.assertEquals(LocalDate.of(2026, 8, 2), row[10]); + Assertions.assertEquals(timestamp, row[11]); + // TIMESTAMPTZ is carried as the UTC wall clock of the instant, not the session zone's. + Assertions.assertEquals(LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC), row[12]); + // Content of the two binary columns is asserted separately, through the STRING mapping: the + // VARBINARY read-BACK in java-common (VectorColumn.getBytesVarbinary) decodes a StringView + // layout that nothing writes -- appendVarbinary and BE's _fill_varbinary_column both use + // [len:long][addr:long] -- so it cannot be used to check a value. What it does show here is + // that the column arrived non-null, and the all-null row below that it can be null. + Assertions.assertNotNull(row[13]); + Assertions.assertNotNull(row[14]); + + for (int i = 0; i < row.length; i++) { + Assertions.assertNull(rows[1][i], "column " + i + " of the all-null row came back non-null"); + } + } + + /** + * Fluss BYTES maps to Doris STRING unless the catalog opts into VARBINARY, and BINARY is + * fixed-width where BYTES is not — so which fluss accessor a column needs is a property of the + * fluss type, not of the Doris one it was mapped to. Both are read here through the STRING + * mapping, which is also the only read-back path that can check a byte value at all (see the + * java-common note in the round-trip test above). + */ + @Test + public void binaryAndBytesReadAsStringsGiveTheirBytes() throws Exception { + TablePath tablePath = TablePath.of(db, "bytes_as_string"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_bytes", DataTypes.BYTES()) + .column("c_binary", DataTypes.BINARY(2)) + .column("c_char", DataTypes.CHAR(2)) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + appendRows(tablePath, GenericRow.of(new byte[] {(byte) 'h', (byte) 'i'}, + new byte[] {(byte) 'o', (byte) 'k'}, BinaryString.fromString("ab"))); + + Object[][] rows = scanAll(tablePath, + columns("c_bytes", "string", "c_binary", "string", "c_char", "string"), 0, 1, 1024); + + Assertions.assertArrayEquals(new Object[] {"hi", "ok", "ab"}, rows[0]); + } + + @Test + public void nestedTypesComeBackWithTheirStructure() throws Exception { + TablePath tablePath = TablePath.of(db, "nested_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_array", DataTypes.ARRAY(DataTypes.INT())) + .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("c_struct", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT()), + DataTypes.FIELD("b", DataTypes.STRING()))) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + + Map map = new LinkedHashMap<>(); + map.put(BinaryString.fromString("k1"), 1); + map.put(BinaryString.fromString("k2"), 2); + appendRows(tablePath, + GenericRow.of(new GenericArray(new int[] {10, 20, 30}), new GenericMap(map), + GenericRow.of(7, BinaryString.fromString("inner"))), + // A shorter array right after a longer one: the value wrappers are cached by position + // and reused across rows, so a stale wrapper would surface as a phantom fourth element. + GenericRow.of(new GenericArray(new int[] {40}), + new GenericMap(Collections.singletonMap(BinaryString.fromString("k3"), 3)), + GenericRow.of(8, BinaryString.fromString("other")))); + + Object[][] rows = scanAll(tablePath, columns( + "c_array", "array", + "c_map", "map", + "c_struct", "struct"), 0, 2, 1024); + + Assertions.assertEquals(2, rows.length); + Assertions.assertEquals("[10, 20, 30]", String.valueOf(rows[0][0])); + Assertions.assertEquals("{k1=1, k2=2}", String.valueOf(rows[0][1])); + Assertions.assertEquals("{a=7, b=inner}", String.valueOf(rows[0][2])); + Assertions.assertEquals("[40]", String.valueOf(rows[1][0])); + Assertions.assertEquals("{k3=3}", String.valueOf(rows[1][1])); + Assertions.assertEquals("{a=8, b=other}", String.valueOf(rows[1][2])); + } + + // ---------------------------------------------------------------- projection and bounds + + /** + * The projection is by name, in the order Doris asked for — which is not the table's order. Passing + * the table's own order through instead would silently transpose columns of the same type. + */ + @Test + public void projectionFollowsTheRequestedNamesAndOrder() throws Exception { + TablePath tablePath = TablePath.of(db, "projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("a", DataTypes.INT()) + .column("b", DataTypes.INT()) + .column("c", DataTypes.INT()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + appendRows(tablePath, GenericRow.of(1, 2, 3)); + + Object[][] rows = scanAll(tablePath, columns("c", "int", "a", "int"), 0, 1, 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {3, 1}, rows[0]); + } + + /** The range is half-open: rows at or past the stopping offset belong to nobody's scan yet. */ + @Test + public void theStoppingOffsetBoundsTheRead() throws Exception { + TablePath tablePath = TablePath.of(db, "bounded"); + createIntTable(tablePath); + appendInts(tablePath, 0, 10); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 4, 1024); + + Assertions.assertEquals(4, rows.length); + Assertions.assertArrayEquals(new Object[] {0}, rows[0]); + Assertions.assertArrayEquals(new Object[] {3}, rows[3]); + } + + @Test + public void theStartingOffsetSkipsWhatCameBefore() throws Exception { + TablePath tablePath = TablePath.of(db, "start_offset"); + createIntTable(tablePath); + appendInts(tablePath, 0, 10); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 6, 10, 1024); + + Assertions.assertEquals(4, rows.length); + Assertions.assertArrayEquals(new Object[] {6}, rows[0]); + Assertions.assertArrayEquals(new Object[] {9}, rows[3]); + } + + /** Rows written after planning are past the stopping offset and must not leak into the scan. */ + @Test + public void rowsWrittenAfterPlanningAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "late_rows"); + createIntTable(tablePath); + appendInts(tablePath, 0, 5); + // Planning would have stopped at 5; these arrive afterwards, inside the same fetched batch range. + appendInts(tablePath, 100, 105); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 5, 1024); + + Assertions.assertEquals(5, rows.length); + Assertions.assertArrayEquals(new Object[] {4}, rows[4]); + } + + /** Doris pulls fixed-size batches; the range's rows must arrive whole across several of them. */ + @Test + public void rangeLargerThanOneBatchIsReadAcrossBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "batched"); + createIntTable(tablePath); + appendInts(tablePath, 0, 25); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 25, 10); + + Assertions.assertEquals(25, rows.length); + for (int i = 0; i < 25; i++) { + Assertions.assertArrayEquals(new Object[] {i}, rows[i], "row " + i); + } + } + + /** + * A partitioned table is addressed by partition id plus bucket. Subscribing without the partition + * would read the wrong bucket entirely. + */ + @Test + public void partitionedTableIsReadThroughItsPartitionId() throws Exception { + TablePath tablePath = TablePath.of(db, "partitioned"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS) + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + writer.append(GenericRow.of(1, BinaryString.fromString("2026_08_02"))); + writer.append(GenericRow.of(2, BinaryString.fromString("2026_08_03"))); + writer.flush(); + } + long partitionId = admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals("2026_08_03")) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + + Map params = params(tablePath, columns("id", "int"), 0, 1); + params.put("fluss.partition_id", String.valueOf(partitionId)); + Object[][] rows = runScanner(params, 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {2}, rows[0]); + } + + // ---------------------------------------------------------------- lifecycle and fail-loud + + /** + * A fluss connection owns netty and metadata-updater threads that outlive the query if the scanner + * leaks it — and a BE runs scanners for the life of the process. + */ + @Test + public void closingTheScannerReleasesItsClientThreads() throws Exception { + TablePath tablePath = TablePath.of(db, "threads"); + createIntTable(tablePath); + appendInts(tablePath, 0, 3); + int before = countClientThreads(); + + for (int i = 0; i < 3; i++) { + scanAll(tablePath, columns("id", "int"), 0, 3, 1024); + } + + long deadline = System.currentTimeMillis() + 30_000; + while (countClientThreads() > before && System.currentTimeMillis() < deadline) { + Thread.sleep(100); + } + Assertions.assertTrue(countClientThreads() <= before, + "fluss client threads leaked: " + before + " before, " + countClientThreads() + " after"); + } + + /** An unreadable range must name what is wrong, not hand fluss a null and fail somewhere inside. */ + @Test + public void missingParameterIsNamed() { + Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); + params.remove("fluss.bucket_id"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params).open()); + Assertions.assertTrue(e.getMessage().contains("fluss.bucket_id"), e.getMessage()); + } + + /** Primary-key and union ranges are planned by FE but not readable here yet. */ + @Test + public void anUnsupportedRangeTypeIsRefused() { + Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); + params.put("fluss.range_type", "PK_FULL"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(e.getMessage().contains("PK_FULL"), e.getMessage()); + } + + /** A column dropped between planning and reading must say so, not read a neighbouring column. */ + @Test + public void columnThatIsNoLongerThereIsNamed() throws Exception { + TablePath tablePath = TablePath.of(db, "missing_column"); + createIntTable(tablePath); + appendInts(tablePath, 0, 1); + + Exception e = Assertions.assertThrows(Exception.class, + () -> runScanner(params(tablePath, columns("nope", "int"), 0, 1), 1024)); + Assertions.assertTrue(rootMessage(e).contains("nope"), rootMessage(e)); + } + + // ---------------------------------------------------------------- helpers + + private static String rootMessage(Throwable e) { + StringBuilder message = new StringBuilder(); + for (Throwable t = e; t != null; t = t.getCause()) { + message.append(t.getMessage()).append(" | "); + } + return message.toString(); + } + + private static int countClientThreads() { + int count = 0; + for (Thread thread : Thread.getAllStackTraces().keySet()) { + // Client-side only: the embedded cluster's own threads are fluss-netty-server-*. + if (thread.getName().startsWith("fluss-netty-client")) { + count++; + } + } + return count; + } + + private void createIntTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(BUCKETS) + .build(), true).get(); + } + + private void appendInts(TablePath tablePath, int fromInclusive, int toExclusive) throws Exception { + GenericRow[] rows = new GenericRow[toExclusive - fromInclusive]; + for (int i = fromInclusive; i < toExclusive; i++) { + rows[i - fromInclusive] = GenericRow.of(i); + } + appendRows(tablePath, rows); + } + + private static void appendRows(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (GenericRow row : rows) { + writer.append(row); + } + writer.flush(); + } + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** The merged map BE hands the scanner: the scan-level properties plus this range's. */ + private static Map params(TablePath tablePath, Map columns, + long startOffset, long stopOffset) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "LOG"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.log_start_offset", String.valueOf(startOffset)); + params.put("fluss.log_stop_offset", String.valueOf(stopOffset)); + params.put("time_zone", "UTC"); + return params; + } + + private static Object[][] scanAll(TablePath tablePath, Map columns, + long startOffset, long stopOffset, int batchSize) throws Exception { + return runScanner(params(tablePath, columns, startOffset, stopOffset), batchSize); + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced. {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; + * reading it as rows would silently compare a column against a row and mostly pass on square data. + */ + private static Object[][] runScanner(Map params, int batchSize) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } +} diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index 075b59cc8d6c23..a76f34c71493e1 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -28,6 +28,7 @@ under the License. java-udf jdbc-scanner paimon-scanner + fluss-scanner max-compute-connector diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml index 4ecf7e35eb830c..43e28fb26fb57f 100644 --- a/fe/fe-connector/fe-connector-fluss/pom.xml +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -64,10 +64,12 @@ under the License. this jar, and a second copy on the child-first plugin classpath is a split-brain risk. - frocksdbjni CANNOT be excluded: it is not a transitive dependency, so there is nothing to exclude. The FE carries it even though only the BE-side scanner ever opens a kv snapshot. - The one thing fluss-client does NOT bring is Arrow: it declares arrow-vector and - arrow-memory-netty `provided`, so a consumer that reaches fluss's ARROW log format has to add - them itself. This module does not (metadata and split planning stay off that path); the - BE-side scanner, which decodes log records, is expected to need them. + Arrow needs no declaration either, despite fluss-client's pom marking arrow-vector and + arrow-memory-netty `provided`: fluss does not use those coordinates. Its ARROW log format + goes through a relocated copy bundled in this same jar (org.apache.fluss.shaded.arrow.**), + allocator included, so a consumer that decodes log records is already served. What such a + consumer DOES need is the add-opens=java.base/java.nio=ALL-UNNAMED JVM flag, which arrow's + off-heap MemoryUtil requires on JDK 9+ (bin/start_be.sh already passes it). --> org.apache.fluss From a242fb16f6ad4e116f6517ab2da682277f426228 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 22:28:32 +0800 Subject: [PATCH 08/35] [feat](fluss) Route a fluss scan range to the BE java scanner FileScannerV2 gains a fluss table format. The reader holds no fluss logic: it merges the scan-level fluss_properties with the per-range fluss_params - the range wins, being the specific half - and hands the result to org.apache.doris.fluss.FlussJniScanner. Partition columns are left out of what the scanner is asked for. FE declares them as path_partition_keys and ships their values on each range, so reading them per row would repeat what the split already states once, and would lay the block out differently from the paimon half of a union read. Only FileScannerV2 is wired. With enable_file_scanner_v2=false the legacy scanner reports "Not supported create reader for table format: fluss", so a fluss regression suite has to pin the session variable - the fuzzy session mode randomizes it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- be/src/exec/scan/file_scanner_v2.cpp | 6 +- be/src/format_v2/jni/fluss_jni_reader.cpp | 124 +++++++++++ be/src/format_v2/jni/fluss_jni_reader.h | 51 +++++ .../format_v2/jni/fluss_jni_reader_test.cpp | 193 ++++++++++++++++++ 4 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 be/src/format_v2/jni/fluss_jni_reader.cpp create mode 100644 be/src/format_v2/jni/fluss_jni_reader.h create mode 100644 be/test/format_v2/jni/fluss_jni_reader_test.cpp diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 4d513acbb9dd4c..3866b42f9f7b76 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -50,6 +50,7 @@ #include "format/format_common.h" #include "format/table/iceberg_scan_semantics.h" #include "format_v2/column_mapper.h" +#include "format_v2/jni/fluss_jni_reader.h" #include "format_v2/jni/iceberg_sys_table_reader.h" #include "format_v2/jni/jdbc_reader.h" #include "format_v2/jni/max_compute_jni_reader.h" @@ -131,7 +132,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { (params.file_format == "parquet" || params.file_format == "orc"); } return table_format == "jdbc" || table_format == "iceberg" || table_format == "hudi" || - table_format == "max_compute" || table_format == "trino_connector"; + table_format == "max_compute" || table_format == "trino_connector" || + table_format == "fluss"; } bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { @@ -608,6 +610,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(); } else if (table_format == "hudi") { *reader = std::make_unique(); + } else if (table_format == "fluss") { + *reader = std::make_unique(); } else if (table_format == "jdbc") { *reader = std::make_unique(); } else if (table_format == "max_compute") { diff --git a/be/src/format_v2/jni/fluss_jni_reader.cpp b/be/src/format_v2/jni/fluss_jni_reader.cpp new file mode 100644 index 00000000000000..12f4b545984a3b --- /dev/null +++ b/be/src/format_v2/jni/fluss_jni_reader.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/fluss_jni_reader.h" + +#include "core/block/block.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" + +namespace doris::format::fluss { + +Status FlussJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for fluss jni reader"); + } + if (!range.table_format_params.__isset.fluss_params || + range.table_format_params.fluss_params.empty()) { + return Status::InternalError( + "missing fluss_params for fluss jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (_scan_params == nullptr || !_scan_params->__isset.fluss_properties || + _scan_params->fluss_properties.empty()) { + return Status::InternalError( + "missing fluss_properties for fluss jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + return Status::OK(); +} + +std::string FlussJniReader::connector_class() const { + return "org/apache/doris/fluss/FlussJniScanner"; +} + +Status FlussJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + DORIS_CHECK(_scan_params != nullptr); + // Scan level first, then the range: the range is the more specific of the two, and it is the one + // that says which bucket and offsets to read. No key is transcribed - both maps are written by FE + // and read by the Java scanner, so naming them here would only add a place to drift. + *params = _scan_params->fluss_properties; + for (const auto& kv : _current_range.table_format_params.fluss_params) { + (*params)[kv.first] = kv.second; + } + return Status::OK(); +} + +Status FlussJniReader::build_jni_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + // A fluss row physically carries its partition column values, but FE declares the partition + // keys as path_partition_keys and ships them per range, so they are constants here. Asking + // the scanner for them would read per row what the split already states once - and would + // read them differently from the legacy JNI path, whose file slots exclude them outright. + if (table_column.is_partition_key && + find_partition_value(table_column, _partition_values) != nullptr) { + continue; + } + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = table_column.type, + .replace_type = "not_replace", + }); + } + return Status::OK(); +} + +Status FlussJniReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + const auto original_rows = *rows; + + const auto& columns = jni_columns(); + DORIS_CHECK(columns.size() == jni_block->columns()); + for (size_t i = 0; i < columns.size(); ++i) { + const auto& column = columns[i]; + DORIS_CHECK(column.output_index < output_block->columns()); + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + + // The columns build_jni_columns() left out: materialized from the range instead of read. + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + const auto* partition_value = find_partition_value(table_column, _partition_values); + if (!table_column.is_partition_key || partition_value == nullptr) { + continue; + } + output_block->get_by_position(i).type = table_column.type; + output_block->replace_by_position( + i, table_column.type->create_column_const(original_rows, *partition_value)); + } + DORIS_CHECK(output_block->rows() == original_rows); + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +} // namespace doris::format::fluss diff --git a/be/src/format_v2/jni/fluss_jni_reader.h b/be/src/format_v2/jni/fluss_jni_reader.h new file mode 100644 index 00000000000000..7c1d31e6ddc8c5 --- /dev/null +++ b/be/src/format_v2/jni/fluss_jni_reader.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { + +/** + * Reads one fluss scan range through org.apache.doris.fluss.FlussJniScanner. + * + * This layer holds no fluss logic. FE writes two untyped string maps - the scan-level + * `fluss_properties` (connection, table identity, client options) and the per-range `fluss_params` + * (which bucket, which offsets) - and the only decision made here is that the range wins where both + * set a key. + */ +class FlussJniReader final : public format::JniTableReader { +public: + ~FlussJniReader() override = default; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + Status build_jni_columns( + std::vector* columns) const override; + Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) override; +}; + +} // namespace doris::format::fluss diff --git a/be/test/format_v2/jni/fluss_jni_reader_test.cpp b/be/test/format_v2/jni/fluss_jni_reader_test.cpp new file mode 100644 index 00000000000000..7f31e9147de9fe --- /dev/null +++ b/be/test/format_v2/jni/fluss_jni_reader_test.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/fluss_jni_reader.h" + +#include + +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { +namespace { + +TFileRangeDesc make_fluss_range(std::map fluss_params) { + TFileRangeDesc range; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + table_format_params.__set_fluss_params(std::move(fluss_params)); + range.__set_table_format_params(std::move(table_format_params)); + return range; +} + +TFileScanRangeParams make_scan_params(std::map fluss_properties) { + TFileScanRangeParams scan_params; + scan_params.__set_fluss_properties(std::move(fluss_properties)); + return scan_params; +} + +Status init_reader(FlussJniReader* reader, TFileScanRangeParams* scan_params, + std::vector projected_columns = {}) { + return reader->init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = FileFormat::JNI, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + }); +} + +Status build_params(FlussJniReader* reader, const TFileRangeDesc& range, + std::map* params) { + reader->_current_range = range; + return reader->build_scanner_params(params); +} + +ColumnDefinition make_column(const std::string& name, DataTypePtr type, bool is_partition_key) { + ColumnDefinition column; + column.name = name; + column.type = std::move(type); + column.is_partition_key = is_partition_key; + return column; +} + +// Both maps reach the Java scanner as one. Dropping either half is invisible in C++ - the scanner +// would just report a missing key - so the merge itself is what has to be pinned here. +TEST(FlussJniReaderTest, MergesScanLevelAndRangeLevelParams) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}, + {"fluss.table_name", "log_basic"}, + {"fluss.client.bootstrap.servers", "host:9123"}}); + auto range = make_fluss_range({{"fluss.range_type", "LOG"}, + {"fluss.bucket_id", "3"}, + {"fluss.log_start_offset", "-2"}, + {"fluss.log_stop_offset", "17"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + ASSERT_TRUE(reader.validate_scan_range(range).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["fluss.db_name"], "db"); + EXPECT_EQ(params["fluss.table_name"], "log_basic"); + EXPECT_EQ(params["fluss.client.bootstrap.servers"], "host:9123"); + EXPECT_EQ(params["fluss.range_type"], "LOG"); + EXPECT_EQ(params["fluss.bucket_id"], "3"); + EXPECT_EQ(params["fluss.log_start_offset"], "-2"); + EXPECT_EQ(params["fluss.log_stop_offset"], "17"); + EXPECT_EQ(params.size(), 7); +} + +// The range is the specific half of the pair. If the scan level ever won, every bucket of a scan +// would read the same one - a query that returns wrong rows rather than failing. +TEST(FlussJniReaderTest, RangeLevelParamWinsOverScanLevelParam) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}, {"fluss.bucket_id", "0"}}); + auto range = make_fluss_range({{"fluss.range_type", "LOG"}, {"fluss.bucket_id", "7"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["fluss.bucket_id"], "7"); +} + +TEST(FlussJniReaderTest, RejectsRangeWithoutFlussParams) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + TFileRangeDesc no_table_format; + EXPECT_FALSE(reader.validate_scan_range(no_table_format).ok()); + + TFileRangeDesc no_fluss_params; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + no_fluss_params.__set_table_format_params(std::move(table_format_params)); + EXPECT_FALSE(reader.validate_scan_range(no_fluss_params).ok()); + + EXPECT_FALSE(reader.validate_scan_range(make_fluss_range({})).ok()); +} + +TEST(FlussJniReaderTest, RejectsScanWithoutFlussProperties) { + auto range = make_fluss_range({{"fluss.range_type", "LOG"}}); + + TFileScanRangeParams empty_scan_params; + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &empty_scan_params).ok()); + EXPECT_FALSE(reader.validate_scan_range(range).ok()); + + auto empty_properties = make_scan_params({}); + FlussJniReader reader_with_empty_properties; + ASSERT_TRUE(init_reader(&reader_with_empty_properties, &empty_properties).ok()); + EXPECT_FALSE(reader_with_empty_properties.validate_scan_range(range).ok()); +} + +// A partition column is a constant of the range, not something the scanner reads. Asking for it +// would also disagree with the paimon half of a union read, whose partition columns come from the +// same per-range values - the two halves of one scan node have to lay their blocks out alike. +TEST(FlussJniReaderTest, LeavesPartitionColumnsToTheRangeAndKeepsOutputPositions) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + std::vector projected_columns { + make_column("id", std::make_shared(), false), + make_column("dt", std::make_shared(), true), + make_column("name", std::make_shared(), false)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(projected_columns)).ok()); + reader._partition_values.emplace("dt", + Field::create_field(std::string("20260101"))); + + std::vector columns; + ASSERT_TRUE(reader.build_jni_columns(&columns).ok()); + ASSERT_EQ(columns.size(), 2); + EXPECT_EQ(columns[0].java_name, "id"); + EXPECT_EQ(columns[0].output_index, 0); + EXPECT_EQ(columns[1].java_name, "name"); + // Its position in the output block, not its position among the columns the scanner returns: + // the two differ by every partition column dropped before it. + EXPECT_EQ(columns[1].output_index, 2); +} + +// A partition column the range has no value for would otherwise leave a hole in the output block. +TEST(FlussJniReaderTest, StillReadsAPartitionColumnTheRangeDidNotCarry) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + std::vector projected_columns { + make_column("id", std::make_shared(), false), + make_column("dt", std::make_shared(), true)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(projected_columns)).ok()); + + std::vector columns; + ASSERT_TRUE(reader.build_jni_columns(&columns).ok()); + ASSERT_EQ(columns.size(), 2); + EXPECT_EQ(columns[1].java_name, "dt"); + EXPECT_EQ(columns[1].output_index, 1); +} + +} // namespace +} // namespace doris::format::fluss From 59e9fc7c44c872f266dd560f2c64f7f57f607cba Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 2 Aug 2026 22:39:04 +0800 Subject: [PATCH 09/35] [feat](fluss) Add the end-to-end suite for fluss log tables Reads the docker fixtures through a fluss catalog: whole table, projection in a different order than the schema, predicates, every mapped type including the all-NULL row, and the partition columns BE materializes from each range rather than reads. Two shapes the planner has to get right are pinned through the plan: pruning a partition must shrink the ranges, not just the partition=N/M line, and a table that was never written to must plan no range at all. The fixture set gains an empty table for the latter. Values are asserted explicitly instead of through a .out baseline, so an expectation and the fixture that produces it stay in one file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 1 + .../docker-compose/fluss/sql/init.sql | 11 + .../fluss/test_fluss_catalog.groovy | 2 +- .../fluss/test_fluss_log_table.groovy | 230 ++++++++++++++++++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index 0688660bee8ace..3389655713f874 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -82,6 +82,7 @@ path string. | `log_basic` | log table, 3 rows, table and column comments | | `log_types` | log table, one column per mapped fluss type, plus an all-NULL row | | `log_part` | log table partitioned by `dt`, partitions `20260101`, `20260102`, `20260103` | +| `log_empty` | log table with no rows at all (planning must emit zero scan ranges) | | `pk_basic` | primary-key table, one updated row and one deleted row | | `pk_types` | primary-key table with the same type coverage as `log_types` | diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index 71b67269429305..41d1274971079a 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -168,6 +168,17 @@ INSERT INTO log_part VALUES (3, 'p2a', '20260102'), (4, 'p3a', '20260103'); +-- --------------------------------------------------------------------------- +-- log_empty: never written to. Every bucket's latest offset is 0, so planning +-- must emit no scan range at all rather than ranges that read nothing. +-- --------------------------------------------------------------------------- +CREATE TABLE log_empty ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '2' +); + -- --------------------------------------------------------------------------- -- pk_basic: primary-key table. Row 2 is updated and row 3 deleted, so a -- correct read returns the merged view, not the raw change log. diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy index fb280546a900c4..3270bd77361115 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -51,7 +51,7 @@ suite("test_fluss_catalog", "p0,external") { def tableRows = sql """show tables""" def tables = tableRows.collect { it[0] } - for (String expected : ["log_basic", "log_types", "log_part", "pk_basic", "pk_types"]) { + for (String expected : ["log_basic", "log_types", "log_part", "log_empty", "pk_basic", "pk_types"]) { assertTrue(tables.contains(expected), "table ${expected} missing: ${tables}") } diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy new file mode 100644 index 00000000000000..38d907d468e3a6 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading fluss log tables end to end: FE planning, the untyped payload it ships, +// the BE java scanner and the partition columns BE materializes from each range. +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are +// static - this suite never writes, so no polling gate is needed. +// +// Everything is asserted explicitly rather than through qt_ and a .out file. The +// values are the fixture's own literals, so an expectation that drifts from the +// fixture is a diff in one file rather than a regenerated baseline nobody reads. +suite("test_fluss_log_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_log_table" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only; the legacy one answers + // "Not supported create reader for table format: fluss". Fuzzy sessions randomize + // this variable, so pinning it is what keeps the suite from failing on half the + // CI runs for a reason that has nothing to do with fluss. + sql """set enable_file_scanner_v2 = true""" + + def scalarOf = { String query -> sql(query)[0][0].toString() } + def planOf = { String query -> + def planRows = sql("""explain ${query}""") + return planRows.collect { it[0].toString() }.join("\n") + } + def logRangesOf = { String plan -> + def matcher = (plan =~ /logRanges=(\d+)/) + assertTrue(matcher.find(), "plan has no flussScan line: ${plan}") + return matcher.group(1) as int + } + + // --- the whole table ---------------------------------------------------- + def basicRows = sql """select id, name, price from log_basic order by id""" + assertEquals(3, basicRows.size()) + assertEquals(["1", "alice", "10.10"], basicRows[0].collect { it.toString() }) + assertEquals(["2", "bob", "20.20"], basicRows[1].collect { it.toString() }) + assertEquals(["3", "carol", "30.30"], basicRows[2].collect { it.toString() }) + + // COUNT(*) projects no column at all: the scanner has to report how many rows it + // read without returning one. A scanner that answered with an empty batch instead + // would make an untouched-looking table out of a populated one. + assertEquals("3", scalarOf("""select count(*) from log_basic""")) + + // --- projection and predicates ----------------------------------------- + def names = sql """select name from log_basic order by name""" + assertEquals(["alice", "bob", "carol"], names.collect { it[0].toString() }) + + // Columns asked for in an order other than the table's: the scanner resolves the + // projection by name, so a positional shortcut anywhere would swap these two. + def swapped = sql """select name, id from log_basic where id = 2""" + assertEquals(1, swapped.size()) + assertEquals(["bob", "2"], swapped[0].collect { it.toString() }) + + def filtered = sql """select id from log_basic where price > 15.00 order by id""" + assertEquals(["2", "3"], filtered.collect { it[0].toString() }) + assertEquals("1", scalarOf("""select count(*) from log_basic where name = 'bob'""")) + + // --- every mapped type survives the round trip -------------------------- + // Asserted through predicates rather than by comparing rendered values: the + // rendering of a map or a struct is the display layer's business, while what this + // suite is about is whether the bytes fluss returned decoded into the right value. + assertEquals("3", scalarOf("""select count(*) from log_types""")) + + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 1 + and f_boolean = true + and f_tinyint = 1 + and f_smallint = 2 + and f_int = 3 + and f_bigint = 4 + and f_float = cast(1.5 as float) + and f_double = 2.5 + and f_decimal = 123.4567 + """)) + + // Fluss BYTES and BINARY map to a Doris string by default, so their content is + // compared as hex; a decoder that lost the length would return a prefix of this. + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 1 + and f_char = 'char1' + and f_string = 'string1' + and hex(f_binary) = '010203' + and hex(f_bytes) = '0A0B' + """)) + + // TIMESTAMP_LTZ is only checked for presence: its rendering depends on the session + // time zone, which is not what this suite is pinning. + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 1 + and f_date = '2026-01-01' + and f_timestamp = '2026-01-01 01:02:03.456789' + and f_timestamp_ltz is not null + """)) + + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 1 + and array_size(f_array) = 3 + and f_array[1] = 1 + and f_array[3] = 3 + and f_map['k1'] = 1 + and f_map['k2'] = 2 + and struct_element(f_row, 'r_int') = 1 + and struct_element(f_row, 'r_string') = 'nested1' + """)) + + // The second row is the negative-value one: a sign lost in decoding shows up here + // and nowhere else. + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 2 + and f_boolean = false + and f_tinyint = -1 + and f_smallint = -2 + and f_int = -3 + and f_bigint = -4 + and f_float = cast(-1.5 as float) + and f_double = -2.5 + and f_decimal = -123.4567 + and array_size(f_array) = 2 + """)) + + // The all-NULL row. Read through a null map that is off by one column, every value + // after it shifts, so this is checked column by column rather than by row count. + assertEquals("1", scalarOf(""" + select count(*) from log_types where id = 3 + and f_boolean is null and f_tinyint is null and f_smallint is null + and f_int is null and f_bigint is null and f_float is null + and f_double is null and f_decimal is null and f_char is null + and f_string is null and f_binary is null and f_bytes is null + and f_date is null and f_timestamp is null and f_timestamp_ltz is null + and f_array is null and f_map is null and f_row is null + """)) + + // --- partitioned table -------------------------------------------------- + // The partition column is not read by the scanner: FE declares it and BE fills it + // in from each range. Checking it against the row it belongs to is what catches a + // partition value attached to the wrong split. + def partRows = sql """select id, name, dt from log_part order by id""" + assertEquals(4, partRows.size()) + assertEquals(["1", "p1a", "20260101"], partRows[0].collect { it.toString() }) + assertEquals(["2", "p1b", "20260101"], partRows[1].collect { it.toString() }) + assertEquals(["3", "p2a", "20260102"], partRows[2].collect { it.toString() }) + assertEquals(["4", "p3a", "20260103"], partRows[3].collect { it.toString() }) + + // Projecting nothing but the partition column leaves the scanner with an empty + // projection - it still has to report the row count for each range. + def perPartition = sql """select dt, count(*) from log_part group by dt order by dt""" + assertEquals(3, perPartition.size()) + assertEquals(["20260101", "2"], perPartition[0].collect { it.toString() }) + assertEquals(["20260102", "1"], perPartition[1].collect { it.toString() }) + assertEquals(["20260103", "1"], perPartition[2].collect { it.toString() }) + + def prunedRows = sql """select id from log_part where dt = '20260101' order by id""" + assertEquals(["1", "2"], prunedRows.collect { it[0].toString() }) + + // --- planning is visible in the plan ------------------------------------ + def basicPlan = planOf("""select * from log_basic""") + assertTrue(basicPlan.contains("flussScan: unionRead=no"), + "no fluss scan line: ${basicPlan}") + assertTrue(basicPlan.contains("lakeSplits=0"), "unexpected lake splits: ${basicPlan}") + assertTrue(basicPlan.contains("mode=auto"), "unexpected union read mode: ${basicPlan}") + // One range per bucket that holds data. Which of the three buckets a fluss log row + // lands in is the writer's choice, so the count is bounded rather than fixed. + def basicRanges = logRangesOf(basicPlan) + assertTrue(basicRanges >= 1 && basicRanges <= 3, + "log_basic planned ${basicRanges} ranges over 3 buckets") + + def fullPartPlan = planOf("""select * from log_part""") + assertTrue(fullPartPlan.contains("partition=3/3"), + "all three partitions should be scanned: ${fullPartPlan}") + def fullPartRanges = logRangesOf(fullPartPlan) + assertTrue(fullPartRanges >= 3, "every partition holds data: ${fullPartPlan}") + + def prunedPlan = planOf("""select * from log_part where dt = '20260101'""") + assertTrue(prunedPlan.contains("partition=1/3"), + "partition pruning did not reach the connector: ${prunedPlan}") + // Pruning has to shrink the work, not just the plan line: a partition name rendered + // one way for the listing and another way for the match would prune to nothing here + // while still reporting 1/3. + def prunedRanges = logRangesOf(prunedPlan) + assertTrue(prunedRanges >= 1 && prunedRanges <= 2, + "one partition has 2 buckets, planned ${prunedRanges} ranges: ${prunedPlan}") + + // --- a table that was never written to ---------------------------------- + // Its buckets stop at offset 0, so planning emits no range at all. The empty answer + // has to come from that, not from a scanner opened on an empty bucket. + assertEquals("0", scalarOf("""select count(*) from log_empty""")) + def emptyRows = sql """select id, name from log_empty""" + assertEquals(0, emptyRows.size()) + def emptyPlan = planOf("""select * from log_empty""") + // An engine that drops a split-less scan altogether is just as correct as one that + // keeps the node, so what is pinned is that no range was planned either way. + def emptyMatcher = (emptyPlan =~ /logRanges=(\d+)/) + assertTrue(!emptyMatcher.find() || emptyMatcher.group(1) == "0", + "a table that was never written to planned ranges: ${emptyPlan}") + + sql """switch internal""" + sql """drop catalog ${catalogName}""" +} From 8f66082bc0e85df581acc5543afc24a52c9af579 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 01:51:17 +0800 Subject: [PATCH 10/35] [fix](fluss) Depend on the fluss test-jar by classifier The build cache extension takes a dependency's type for its file extension when it hashes a module's inputs, so type "test-jar" sent it looking for a fluss-server-1.0-SNAPSHOT.test-jar that exists in no repository and the frontend build failed before compiling anything. The classifier spelling resolves the very same file. These were the only two modules in the tree depending on a foreign test-jar, which is why nothing caught it earlier: the module tests were always run with the build cache turned off. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- fe/be-java-extensions/fluss-scanner/pom.xml | 4 +++- fe/fe-connector/fe-connector-fluss/pom.xml | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fe/be-java-extensions/fluss-scanner/pom.xml b/fe/be-java-extensions/fluss-scanner/pom.xml index 7f771902d39b36..3ce1b8b339001a 100644 --- a/fe/be-java-extensions/fluss-scanner/pom.xml +++ b/fe/be-java-extensions/fluss-scanner/pom.xml @@ -85,11 +85,13 @@ under the License. test + org.apache.fluss fluss-server ${fluss.version} - test-jar + tests test diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml index 43e28fb26fb57f..0dc9edadab7812 100644 --- a/fe/fe-connector/fe-connector-fluss/pom.xml +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -110,11 +110,18 @@ under the License. test + org.apache.fluss fluss-server ${fluss.version} - test-jar + tests test From d27b62e7499883a883e87f0f6e74f75ccad7d61b Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 01:51:17 +0800 Subject: [PATCH 11/35] [fix](fluss) Read a range whose projection selects no column A query over a partitioned table can need none of the columns the scanner reads: "select dt, count(*) ... group by dt" wants only the partition column, and BE materializes that one from the range itself. Fluss rejects an empty projection outright, so opening the scanner threw and the query failed. Stand the narrowest legal request in for it; nothing reads the column that comes back, since getNext iterates over the required fields and reports the row count alone. count(*) alone did not reach this: the planner keeps the first column as a placeholder slot, so only a projection made entirely of partition columns empties out. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../apache/doris/fluss/FlussJniScanner.java | 14 +++++++++++++- .../doris/fluss/FlussJniScannerLogTest.java | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java index b05150bebe0762..6c8d2ec278bbcd 100644 --- a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java @@ -167,7 +167,7 @@ public void open() throws IOException { projectedTypes.add(rowType.getTypeAt(index)); } - logScanner = table.newScan().project(projection).createLogScanner(); + logScanner = table.newScan().project(scanProjection(projection)).createLogScanner(); subscribe(); } catch (Throwable e) { try { @@ -214,6 +214,18 @@ private int[] projectionOf(RowType rowType) { return projection; } + /** + * What to actually ask fluss for. Normally the projected columns, but a query can need none of + * them — selecting only partition columns leaves this scanner with an empty projection, because + * BE materializes those from the range instead. Fluss rejects an empty projection outright + * ({@code Projection.of}), so the narrowest legal request stands in for it. Nothing reads the + * column that comes back: {@link #getNext} iterates over {@link #fields}, which is empty, and + * reports the row count alone. A fluss table always has a first column. + */ + private static int[] scanProjection(int[] projection) { + return projection.length == 0 ? new int[] {0} : projection; + } + private Configuration clientConfig() { Configuration config = new Configuration(); for (Map.Entry entry : params.entrySet()) { diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java index eb82c5a317d4a7..80b0593fe06c16 100644 --- a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java @@ -292,6 +292,25 @@ public void projectionFollowsTheRequestedNamesAndOrder() throws Exception { Assertions.assertArrayEquals(new Object[] {3, 1}, rows[0]); } + /** + * A query can ask for no column of this scanner at all: {@code select dt, count(*) ... group by dt} + * over a partitioned table needs only the partition column, and BE materializes that one from the + * range. Fluss refuses an empty projection, so the scanner has to stand something in for it and + * still report how many rows the range held — answering 0 rows would turn a populated partition + * into an empty one. + */ + @Test + public void projectingNoColumnStillCountsTheRows() throws Exception { + TablePath tablePath = TablePath.of(db, "no_projection"); + createIntTable(tablePath); + appendInts(tablePath, 0, 7); + + Object[][] rows = scanAll(tablePath, columns(), 0, 7, 1024); + + Assertions.assertEquals(7, rows.length); + Assertions.assertEquals(0, rows[0].length, "a column was returned for an empty projection"); + } + /** The range is half-open: rows at or past the stopping offset belong to nobody's scan yet. */ @Test public void theStoppingOffsetBoundsTheRead() throws Exception { From e6adf9830f5a8fd27919ee74721c799b715ebae4 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 07:05:44 +0800 Subject: [PATCH 12/35] [feat](fluss) Plan a primary-key table read as snapshot plus change log A fluss primary-key table cannot be read the way a log table is: its log is a change log, so replaying it verbatim returns superseded and deleted rows. Each bucket is planned instead as its latest kv snapshot plus the change log that followed it, which the scanner merges by key. A bucket fluss has never snapshotted carries -1 and the earliest sentinel, and its state is rebuilt by replaying the whole change log -- equally correct, only slower. Snapshots are asked for before offsets, and the order is load-bearing: a snapshot committed between the two calls ends past the offset planning stopped at, and that bucket would then be read from a snapshot already containing rows written after the query started while every other bucket stopped where planning saw it. Log offsets only move forward, so this order keeps every snapshot at or behind the stopping offset. EXPLAIN counts primary-key ranges apart from log ranges. The two are read by different code, and a single total cannot tell a primary-key table planned the right way from one planned the wrong way. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussScanPlanProvider.java | 113 +++++-- .../fluss/FlussLogScanPlanClusterTest.java | 17 -- .../fluss/FlussPkScanPlanClusterTest.java | 289 ++++++++++++++++++ .../connector/fluss/FlussSplitPlanTest.java | 238 ++++++++++++++- .../fluss/RecordingFlussAdminOps.java | 20 +- 5 files changed, 630 insertions(+), 47 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java index 7f9b05cf550833..8c66f6dda4df70 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -29,6 +29,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.table.scanner.log.LogScanner; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; @@ -51,14 +52,19 @@ * a bucket planned later would read rows written after the query started while an earlier bucket did * not. A bucket that has never been written to (stopping offset 0) yields no range at all. * - *

Planning only reads metadata — partition lists and offsets — so it is safe to run for an - * {@code EXPLAIN}, which does reach {@code planScan}. (There is no explain-only signal on the SPI in - * this branch; the point is that fluss does not need one. A future change that takes a snapshot lease - * during planning would.) + *

A primary-key table cannot be read that way: its log is a change log, so replaying it verbatim + * returns superseded and deleted rows. Each bucket is read instead as its latest kv snapshot plus the + * change log that followed, which the scanner merges by key. A bucket fluss has never snapshotted is + * rebuilt by replaying its whole change log, which is equally correct and only slower. * - *

What is NOT here yet: primary-key tables, and the union of a table's lake with its log. Both are - * refused loudly rather than served as a partial answer — a datalake table read as fluss-only would - * silently return just the rows that have not been tiered away, which looks like a working query. + *

Planning only reads metadata — partition lists, offsets and snapshot ids — so it is safe to run + * for an {@code EXPLAIN}, which does reach {@code planScan}. (There is no explain-only signal on the + * SPI in this branch; the point is that fluss does not need one. A future change that takes a snapshot + * lease during planning would.) + * + *

What is NOT here yet: the union of a table's lake with its log. It is refused loudly rather than + * served as a partial answer — a datalake table read as fluss-only would silently return just the rows + * that have not been tiered away, which looks like a working query. * {@code fluss.union_read.mode=disabled} is how a user asks for the fluss-only read on purpose. */ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { @@ -90,6 +96,7 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { * either means revisiting this (the ES provider carries the same caveat). */ private int plannedLogRanges; + private int plannedPkRanges; private boolean plannedUnionRead; public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties) { @@ -104,30 +111,61 @@ public List planScan(ConnectorSession session, ConnectorScan FlussConnectorProperties.unionReadMode(catalogProperties); rejectWhatIsNotImplemented(handle, mode); - TablePath tablePath = handle.toTablePath(); List buckets = allBuckets(handle.getBucketCount()); List ranges = new ArrayList<>(); if (handle.isPartitioned()) { for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { // fluss's own partition name ("20260101$cn"), not the Doris one: this is a fluss API. - Map stopping = adminOps.listOffsets( - tablePath, partition.getPartitionName(), buckets, new OffsetSpec.LatestSpec()); - appendLogRanges(ranges, + appendPartitionRanges(ranges, handle, FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), - buckets, stopping); + buckets, partition.getPartitionName()); } } else { - Map stopping = - adminOps.listOffsets(tablePath, buckets, new OffsetSpec.LatestSpec()); - appendLogRanges(ranges, FlussScanRange.Partition.NONE, buckets, stopping); + appendPartitionRanges(ranges, handle, FlussScanRange.Partition.NONE, buckets, null); } - plannedLogRanges = ranges.size(); + plannedLogRanges = count(ranges, FlussScanRange.RangeType.LOG); + plannedPkRanges = count(ranges, FlussScanRange.RangeType.PK_FULL); plannedUnionRead = false; return ranges; } + /** + * The ranges covering one partition of a table, or the whole of an unpartitioned one + * ({@code flussPartitionName} is null, which is also how the two admin overloads are told apart). + */ + private void appendPartitionRanges(List ranges, FlussTableHandle handle, + FlussScanRange.Partition partition, List buckets, String flussPartitionName) { + TablePath tablePath = handle.toTablePath(); + if (!handle.hasPrimaryKey()) { + appendLogRanges(ranges, partition, buckets, + latestOffsets(tablePath, flussPartitionName, buckets)); + return; + } + // Snapshots BEFORE offsets, and the order is load-bearing. A snapshot committed between the two + // calls ends past the offset planning stopped at; that bucket would then be read from a snapshot + // already containing rows written after the query started, while every other bucket stopped + // where planning saw it. Log offsets only move forward, so asking in this order keeps every + // snapshot at or behind the stopping offset. + KvSnapshots snapshots = latestKvSnapshots(tablePath, flussPartitionName); + appendPkRanges(ranges, partition, buckets, snapshots, + latestOffsets(tablePath, flussPartitionName, buckets)); + } + + private KvSnapshots latestKvSnapshots(TablePath tablePath, String flussPartitionName) { + return flussPartitionName == null + ? adminOps.getLatestKvSnapshots(tablePath) + : adminOps.getLatestKvSnapshots(tablePath, flussPartitionName); + } + + private Map latestOffsets(TablePath tablePath, String flussPartitionName, + List buckets) { + return flussPartitionName == null + ? adminOps.listOffsets(tablePath, buckets, new OffsetSpec.LatestSpec()) + : adminOps.listOffsets(tablePath, flussPartitionName, buckets, new OffsetSpec.LatestSpec()); + } + /** * Refuses the reads this connector cannot serve yet, naming what would have to change. Serving them * partially is the failure mode to avoid: a datalake table planned as fluss-only returns whatever @@ -136,11 +174,6 @@ public List planScan(ConnectorSession session, ConnectorScan */ private void rejectWhatIsNotImplemented(FlussTableHandle handle, FlussConnectorProperties.UnionReadMode mode) { - if (handle.hasPrimaryKey()) { - throw new DorisConnectorException("Reading the fluss primary-key table '" - + handle.getDatabaseName() + "." + handle.getTableName() - + "' is not supported yet; only log tables can be read."); - } if (!handle.isDataLakeEnabled() || mode == FlussConnectorProperties.UnionReadMode.DISABLED) { // Not a lake table, or the user asked for the fluss-only read explicitly. return; @@ -203,6 +236,39 @@ private static void appendLogRanges(List ranges, } } + /** + * One range per bucket that holds anything: its latest kv snapshot, plus the change log from where + * that snapshot ended up to where planning saw the log. A bucket fluss has never snapshotted gets + * {@code -1} and the earliest sentinel, and its state is rebuilt by replaying the whole change log + * — correct, because a primary-key table's log carries every change, just slower. + */ + private static void appendPkRanges(List ranges, + FlussScanRange.Partition partition, List buckets, KvSnapshots snapshots, + Map stopping) { + for (int bucket : buckets) { + long snapshotId = snapshots.getSnapshotId(bucket).orElse(FlussScanRange.NO_KV_SNAPSHOT); + long logStart = snapshots.getLogOffset(bucket).orElse(LogScanner.EARLIEST_OFFSET); + Long stop = stopping.get(bucket); + long logStop = stop == null ? 0L : stop; + if (snapshotId == FlussScanRange.NO_KV_SNAPSHOT && logStop <= 0) { + // Nothing snapshotted and nothing logged: the bucket is empty. A bucket WITH a snapshot + // is planned even when its log has caught up, because the snapshot still holds rows. + continue; + } + ranges.add(FlussScanRange.pkFull(partition, bucket, snapshotId, logStart, logStop)); + } + } + + private static int count(List ranges, FlussScanRange.RangeType rangeType) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (((FlussScanRange) range).getRangeType() == rangeType) { + found++; + } + } + return found; + } + private static List allBuckets(int bucketCount) { List buckets = new ArrayList<>(bucketCount); for (int bucket = 0; bucket < bucketCount; bucket++) { @@ -249,7 +315,9 @@ public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { @@ -257,6 +325,7 @@ public void appendExplainInfo(StringBuilder output, String prefix, Map plan("pk_table")); - } - private List plan(String tableName) { FlussTestSession session = new FlussTestSession(1L, "cluster-plan"); ConnectorTableHandle handle = connector.getMetadata(session) diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java new file mode 100644 index 00000000000000..1620080d6bd871 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Primary-key split planning against a real fluss cluster started in this JVM. + * + *

{@link FlussSplitPlanTest} proves the planning logic over recorded snapshot answers; this proves + * the premise those recordings encode — that a cluster really reports a never-snapshotted bucket as + * "no snapshot at all" rather than snapshot 0, and that the log offset it reports alongside a snapshot + * is the one a reader has to resume from. A recording that has drifted from the real answer leaves + * every unit test above green while a primary-key query returns superseded rows. + * + *

Snapshots are triggered, not waited for: the cluster's periodic interval is ten minutes, + * so nothing snapshots on its own and every test states exactly which buckets have one. That is what + * makes "this bucket has a snapshot and a log tail after it" and "this bucket has none" both + * deterministic in the same class. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussPkScanPlanClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands where the test can trigger a snapshot for it. */ + private static final int BUCKETS = 1; + + /** The cluster extension drops every non-built-in database after each test. */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_pk_scan_plan_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "pk_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + } + + /** + * The two halves of a primary-key range have to line up with each other: the snapshot the cluster + * reports, and the change log resumed at exactly where that snapshot ended. Resuming earlier + * replays changes the snapshot already holds; resuming later drops the ones it does not. + */ + @Test + public void snapshottedBucketResumesTheChangeLogWhereTheSnapshotEnded() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_table"); + upsert(tablePath, 1, 2, 3); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), 0)); + // A tail the snapshot does not cover. Nothing snapshots it: the periodic interval is ten + // minutes and this test does not trigger again. + upsert(tablePath, 4, 5); + + List ranges = plan("pk_table"); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath).get(); + long latest = admin.listOffsets(tablePath, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("0", props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(snapshots.getSnapshotId(0).orElse(-1L)), + props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(snapshots.getLogOffset(0).orElse(-2L)), + props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(latest), props.get("fluss.log_stop_offset")); + + // And the fixture really did produce the shape this test is about, rather than passing because + // there was no snapshot and no tail to disagree about. + long logStart = Long.parseLong(props.get("fluss.log_start_offset")); + Assertions.assertTrue(Long.parseLong(props.get("fluss.kv_snapshot_id")) >= 0, props.toString()); + Assertions.assertTrue(logStart > 0, props.toString()); + Assertions.assertTrue(logStart < latest, props.toString()); + } + + /** + * With no snapshot the state has to be rebuilt from the whole change log, and the cluster says so + * by reporting no snapshot id AND no log offset for the bucket. Reading either as a number — 0 for + * the snapshot, 0 for the offset — would silently read a snapshot that does not exist. + */ + @Test + public void bucketFlussHasNeverSnapshottedReplaysItsWholeChangeLog() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_table"); + upsert(tablePath, 1, 2, 3); + + List ranges = plan("pk_table"); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath).get(); + Assertions.assertFalse(snapshots.getSnapshotId(0).isPresent(), "the fixture snapshotted"); + Assertions.assertFalse(snapshots.getLogOffset(0).isPresent(), "the fixture snapshotted"); + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("-1", props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals("-2", props.get("fluss.log_start_offset")); + long latest = admin.listOffsets(tablePath, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + Assertions.assertEquals(String.valueOf(latest), props.get("fluss.log_stop_offset")); + } + + @Test + public void primaryKeyTableNothingWasWrittenToPlansNoRanges() { + Assertions.assertTrue(plan("pk_table").isEmpty()); + } + + /** + * A partitioned primary-key table is snapshotted per partition, so its snapshots have to be asked + * for per partition too. Asking at table level returns the wrong partition's offsets, which reads + * the change log from the wrong place. + */ + @Test + public void partitionedPrimaryKeyRangesCarryTheirOwnPartitionSnapshots() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_part"); + admin.createTable(tablePath, + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsertPartitioned(tablePath, "2026_08_03", 1, 2); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), partitionId, 0)); + upsertPartitioned(tablePath, "2026_08_03", 3); + + List ranges = plan("pk_part"); + + // Only the written partition has anything to read; the empty one yields no range. + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("dt=2026_08_03", props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath, "2026_08_03").get(); + Assertions.assertEquals(String.valueOf(snapshots.getSnapshotId(0).orElse(-1L)), + props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(snapshots.getLogOffset(0).orElse(-2L)), + props.get("fluss.log_start_offset")); + Assertions.assertTrue(Long.parseLong(props.get("fluss.kv_snapshot_id")) >= 0, props.toString()); + } + + private List plan(String tableName) { + FlussTestSession session = new FlussTestSession(1L, "cluster-pk-plan"); + ConnectorTableHandle handle = connector.getMetadata(session) + .getTableHandle(session, db, tableName).orElseThrow(AssertionError::new); + return connector.getScanPlanProvider().planScan(session, + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + + private static long tableId(TablePath tablePath) throws Exception { + return admin.getTableInfo(tablePath).get().getTableId(); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + private static void upsert(TablePath tablePath, long... ids) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (long id : ids) { + writer.upsert(GenericRow.of(id, BinaryString.fromString("name-" + id))); + } + writer.flush(); + } + } + + private static void upsertPartitioned(TablePath tablePath, String partition, long... ids) + throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (long id : ids) { + writer.upsert(GenericRow.of(id, BinaryString.fromString(partition))); + } + writer.flush(); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java index f400ac30b32faa..f40bcf1715fbdf 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -26,6 +26,7 @@ import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.metadata.LakeSnapshot; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -53,6 +54,10 @@ public class FlussSplitPlanTest { private static final TablePath LOG_TABLE = TablePath.of("db", "log_tbl"); + private static final TablePath PK_TABLE = TablePath.of("db", "pk_tbl"); + + /** Fixture shorthand: this bucket has never been snapshotted. */ + private static final long NO_SNAPSHOT = -1L; private RecordingFlussAdminOps adminOps; private ConnectorSession session; @@ -202,21 +207,157 @@ public void anUnpartitionedTableDeclaresNoPartitionKeys() { Assertions.assertFalse(props.containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); } + // ---------------------------------------------------------------- primary-key table + + /** + * A primary-key bucket is read as "the kv snapshot, then the change log that followed it". The + * starting offset therefore has to be the one the snapshot ended at: starting anywhere earlier + * replays changes the snapshot already contains, and anywhere later drops changes it does not. + */ + @Test + public void primaryKeyBucketsAreReadFromTheirSnapshotForward() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {4L, 9L, 2L}, new long[] {40L, 90L, 20L}); + latestOffsets(null, 55L, 90L, 31L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 40L, 55L); + // Nothing new since the snapshot: still planned, and the scanner reads the snapshot alone. + assertPkRange(ranges.get(1), 1, 9L, 90L, 90L); + assertPkRange(ranges.get(2), 2, 2L, 20L, 31L); + } + + /** + * With no snapshot the whole state is rebuilt by replaying the change log from the beginning, + * which is correct because a primary-key table's log carries every change. The two facts that say + * so — {@code -1} and the earliest sentinel — travel separately, and a bucket that got one without + * the other would read a snapshot that does not exist or start from the wrong place. + */ + @Test + public void bucketsWithoutSnapshotsReplayTheirWholeChangeLog() { + registerPkTable(PK_TABLE, 2); + kvSnapshots(null, new long[] {NO_SNAPSHOT, 3L}, new long[] {0L, 30L}); + latestOffsets(null, 12L, 44L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, -1L, -2L, 12L); + assertPkRange(ranges.get(1), 1, 3L, 30L, 44L); + } + + /** Neither snapshotted nor written to: nothing to read, so nothing for BE to open a scanner for. */ + @Test + public void neverWrittenPrimaryKeyBucketsAreSkipped() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {NO_SNAPSHOT, 7L, NO_SNAPSHOT}, new long[] {0L, 70L, 0L}); + latestOffsets(null, 0L, 88L, 5L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 1, 7L, 70L, 88L); + assertPkRange(ranges.get(1), 2, -1L, -2L, 5L); + } + + /** + * A bucket the offsets answer says nothing about still has to be read when it has a snapshot: the + * snapshot holds rows regardless of what the log is doing. Skipping on "no stopping offset" alone + * would drop them, and the query would succeed with a bucket's worth of rows missing. + */ + @Test + public void snapshottedBucketMissingFromTheOffsetsAnswerIsStillRead() { + registerPkTable(PK_TABLE, 2); + kvSnapshots(null, new long[] {6L, NO_SNAPSHOT}, new long[] {60L, 0L}); + Map partialOffsets = new LinkedHashMap<>(); + partialOffsets.put(1, 9L); + adminOps.latestOffsetsByPartition.put(null, partialOffsets); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + // No log to read, so the range covers the snapshot alone. + assertPkRange(ranges.get(0), 0, 6L, 60L, 0L); + assertPkRange(ranges.get(1), 1, -1L, -2L, 9L); + } + + /** + * Snapshots are asked for BEFORE offsets, and the order is the whole point. A snapshot committed + * between the two calls ends past the offset planning stopped at; that bucket would then be read + * from its snapshot — which already contains rows written after the query started — while every + * other bucket stopped where planning saw them. Log offsets only move forward, so asking in this + * order keeps every snapshot at or behind the stopping offset. + */ + @Test + public void snapshotsAreAskedForBeforeTheOffsetsThatBoundThem() { + registerPkTable(PK_TABLE, 1); + kvSnapshots(null, new long[] {1L}, new long[] {10L}); + latestOffsets(null, 20L); + + plan(PK_TABLE, catalog()); + + int snapshotCall = indexOfCall("getLatestKvSnapshots("); + int offsetCall = indexOfCall("listOffsets("); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < offsetCall, adminOps.calls.toString()); + } + + @Test + public void partitionedPrimaryKeyTablesTakeSnapshotsAndOffsetsPerPartition() { + registerPartitionedPkTable(1, "20260101", "20260102"); + kvSnapshots("20260101", new long[] {5L}, new long[] {50L}); + latestOffsets("20260101", 60L); + kvSnapshots("20260102", new long[] {NO_SNAPSHOT}, new long[] {0L}); + latestOffsets("20260102", 8L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertEquals("dt=20260101", ranges.get(0).getProperties().get("fluss.partition_name")); + assertPkRange(ranges.get(0), 0, 5L, 50L, 60L); + Assertions.assertEquals("dt=20260102", ranges.get(1).getProperties().get("fluss.partition_name")); + assertPkRange(ranges.get(1), 0, -1L, -2L, 8L); + } + + /** Pruning applies to a primary-key table exactly as it does to a log table. */ + @Test + public void prunedPartitionsOfPrimaryKeyTablesAreNotEvenAskedAbout() { + registerPartitionedPkTable(1, "20260101", "20260102"); + kvSnapshots("20260102", new long[] {2L}, new long[] {20L}); + latestOffsets("20260102", 25L); + + List ranges = plan(PK_TABLE, catalog(), + Collections.singletonList("dt=20260102")); + + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("getLatestKvSnapshots(")).count(), + adminOps.calls.toString()); + } + // ---------------------------------------------------------------- what is refused, and why + /** + * A tiered primary-key table is refused for the same reason a tiered log table is: the fluss-only + * read returns whatever has not been tiered away yet, which is a successful query with missing rows. + * The primary-key read being implemented does not change that. + */ @Test - public void primaryKeyTableIsRefusedRatherThanReadAsALog() { - TablePath pkTable = TablePath.of("db", "pk_tbl"); - adminOps.tableInfos.put(pkTable, FlussTestTables.builder(pkTable) + public void tieredPrimaryKeyTableIsRefusedUntilTheUnionReadExists() { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) .column("id", DataTypes.INT().copy(false)) .column("v", DataTypes.STRING()) .primaryKey("id") .buckets(2, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") .build()); + adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, - () -> plan(pkTable, catalog())); - Assertions.assertTrue(e.getMessage().contains("primary-key"), e.getMessage()); + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("not supported yet"), e.getMessage()); } /** @@ -321,7 +462,29 @@ public void explainReportsHowTheScanWasActuallyPlanned() { provider.appendExplainInfo(output, " ", Collections.emptyMap()); Assertions.assertEquals( - " flussScan: unionRead=no, lakeSplits=0, logRanges=2, mode=auto\n", output.toString()); + " flussScan: unionRead=no, lakeSplits=0, logRanges=2, pkRanges=0, mode=auto\n", + output.toString()); + } + + /** + * Counted apart from log ranges rather than lumped together: the two are read by different code + * paths, and a test that only sees a total cannot tell a primary-key table that was planned the + * wrong way from one that was planned the right way. + */ + @Test + public void explainCountsPrimaryKeyRangesApartFromLogRanges() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {1L, NO_SNAPSHOT, NO_SNAPSHOT}, new long[] {10L, 0L, 0L}); + latestOffsets(null, 12L, 4L, 0L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog()); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals( + "flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=2, mode=auto\n", + output.toString()); } @Test @@ -414,6 +577,49 @@ private void registerLakeTable(int buckets) { .build()); } + private void registerPkTable(TablePath tablePath, int buckets) { + adminOps.tableInfos.put(tablePath, FlussTestTables.builder(tablePath) + .column("id", DataTypes.INT().copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(buckets, "id") + .build()); + } + + /** A primary-key table partitioned by {@code dt}, with partition ids 100, 101, ... in order. */ + private void registerPartitionedPkTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .primaryKey("id", "dt") + .partitionedBy("dt") + .buckets(buckets, "id") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(PK_TABLE, partitions); + } + + /** + * Latest kv snapshot per bucket 0..n-1 of {@code partitionName} ({@code null} = unpartitioned); + * {@link #NO_SNAPSHOT} for a bucket that has never been snapshotted. Such a bucket gets a null + * snapshot id AND a null log offset, which is the only shape a cluster produces — the client + * asserts the two are both set or both absent when it decodes the response. + */ + private void kvSnapshots(String partitionName, long[] snapshotIds, long[] logOffsets) { + Map ids = new LinkedHashMap<>(); + Map offsets = new LinkedHashMap<>(); + for (int bucket = 0; bucket < snapshotIds.length; bucket++) { + boolean snapshotted = snapshotIds[bucket] != NO_SNAPSHOT; + ids.put(bucket, snapshotted ? snapshotIds[bucket] : null); + offsets.put(bucket, snapshotted ? logOffsets[bucket] : null); + } + adminOps.kvSnapshotsByPartition.put(partitionName, new KvSnapshots(1L, null, ids, offsets)); + } + /** Latest offsets for buckets 0..n-1 of {@code partitionName} ({@code null} = unpartitioned). */ private void latestOffsets(String partitionName, long... offsets) { Map byBucket = new LinkedHashMap<>(); @@ -431,6 +637,26 @@ private static void assertLogRange(ConnectorScanRange range, int bucket, long st Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); } + private static void assertPkRange(ConnectorScanRange range, int bucket, long snapshotId, + long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(snapshotId), props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + + /** Position of the first recorded call starting with {@code prefix}, or -1. */ + private int indexOfCall(String prefix) { + for (int i = 0; i < adminOps.calls.size(); i++) { + if (adminOps.calls.get(i).startsWith(prefix)) { + return i; + } + } + return -1; + } + private static void assertPartition(ConnectorScanRange range, String partitionName, long partitionId, int bucket, long stop) { Map props = range.getProperties(); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java index a6533935b0cb05..77cb83200270ab 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -62,6 +62,11 @@ class RecordingFlussAdminOps implements FlussAdminOps { * unpartitioned table) — what split planning stops each bucket at. */ final Map> latestOffsetsByPartition = new HashMap<>(); + /** + * Latest kv snapshot per bucket, keyed by fluss's own partition name ({@code null} for an + * unpartitioned table) — where primary-key planning starts each bucket's change log. + */ + final Map kvSnapshotsByPartition = new HashMap<>(); /** The readable lake snapshot, or {@code null} to answer as a table nothing has been tiered from. */ LakeSnapshot readableLakeSnapshot; /** When set, every call throws this instead of answering — the "cluster is unreachable" case. */ @@ -138,12 +143,23 @@ public TableStats getTableStats(TablePath tablePath) { @Override public KvSnapshots getLatestKvSnapshots(TablePath tablePath) { - throw notProgrammed("getLatestKvSnapshots"); + return recordedKvSnapshots(tablePath, null); } @Override public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName) { - throw notProgrammed("getLatestKvSnapshots"); + return recordedKvSnapshots(tablePath, partitionName); + } + + private KvSnapshots recordedKvSnapshots(TablePath tablePath, String partitionName) { + calls.add("getLatestKvSnapshots(" + tablePath + + (partitionName == null ? "" : ", " + partitionName) + ")"); + KvSnapshots snapshots = kvSnapshotsByPartition.get(partitionName); + if (snapshots == null) { + throw new IllegalStateException( + "no kv snapshots programmed for partition '" + partitionName + "'"); + } + return snapshots; } @Override From f65d7957be2436d010e2a8245cf61bdf98bc59bf Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 07:19:45 +0800 Subject: [PATCH 13/35] [feat](fluss) Read a primary-key range as a snapshot merged with its log A primary-key table's log is a change log, so replaying it verbatim returns superseded and deleted rows. A PK_FULL range is read instead through fluss's own KvSnapshotAndLogBatchScanner, which merges the bucket's kv snapshot with the change log that followed it, by key. Both reads are now fluss BatchScanners -- the bounded log read moves into one, carrying its three stop conditions verbatim -- so the row loop no longer knows which kind of range it is draining. The projection differs between them and that is deliberate: the log path still needs a stand-in column when the projection is empty, because fluss rejects an empty one, while the primary-key reader appends the key columns to what it fetches and projects the merged row back down, so it never asks for an empty projection at all. The C++ glue needs no change: it merges the two parameter maps and forwards them, so fluss.kv_snapshot_id reaches the scanner without anything in between knowing what it means. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../doris/fluss/BoundedLogBatchScanner.java | 132 ++++ .../apache/doris/fluss/FlussJniScanner.java | 128 ++-- .../doris/fluss/FlussJniScannerLogTest.java | 11 +- .../doris/fluss/FlussJniScannerPkTest.java | 594 ++++++++++++++++++ 4 files changed, 792 insertions(+), 73 deletions(-) create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java create mode 100644 fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java new file mode 100644 index 00000000000000..48d94cc6e842f6 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.utils.CloseableIterator; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * One bucket of a fluss log table over {@code [logStartOffset, logStopOffset)}, as a bounded scanner. + * + *

A fluss log scanner is a streaming reader with no end, so the bound has to be imposed here, and + * reaching it has to be detected three ways — all of them taken from fluss's own bounded reader, + * {@code KvSnapshotAndLogBatchScanner#pollLogRecords}: + *

+ * + *

Shaped as a {@link BatchScanner} so that the log read and the primary-key read + * ({@code KvSnapshotAndLogBatchScanner}, which already is one) present the same interface to + * {@link FlussJniScanner} — the difference between the two belongs here, not in the row loop. + */ +class BoundedLogBatchScanner implements BatchScanner { + + private final LogScanner logScanner; + private final TableBucket tableBucket; + private final long logStopOffset; + + private boolean finished; + + /** + * @param projection table field indexes to read, in the order the caller wants them back; must + * not be empty, which fluss rejects outright + * @param logStartOffset a real offset, or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel + */ + BoundedLogBatchScanner(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset) { + this.tableBucket = tableBucket; + this.logStopOffset = logStopOffset; + LogScanner scanner = table.newScan().project(projection).createLogScanner(); + try { + Long partitionId = tableBucket.getPartitionId(); + if (partitionId == null) { + scanner.subscribe(tableBucket.getBucket(), logStartOffset); + } else { + scanner.subscribe(partitionId, tableBucket.getBucket(), logStartOffset); + } + } catch (RuntimeException | Error e) { + // The scanner is already running its fetcher threads; leaving it unreferenced would keep + // them alive for the life of the BE process. + try { + scanner.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + this.logScanner = scanner; + } + + @Override + public CloseableIterator pollBatch(Duration timeout) { + if (finished) { + return null; + } + ScanRecords scanRecords = logScanner.poll(timeout); + List rows = new ArrayList<>(); + for (ScanRecord record : scanRecords.records(tableBucket)) { + long offset = record.logOffset(); + if (offset >= logStopOffset) { + // Past the end of this range: another query's rows, not ours. + finished = true; + break; + } + rows.add(record.getRow()); + if (offset >= logStopOffset - 1) { + // The last record of the range. Do not poll again: the record AT the stopping offset + // may not exist, and waiting for it never returns. + finished = true; + break; + } + } + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { + // The fetch reached the end of the range without necessarily yielding a record there — the + // tail can be control records, which take offsets but are never scanned. Without this the + // loop would poll for a row that is never coming. + finished = true; + } + return CloseableIterator.wrap(rows.iterator()); + } + + @Override + public void close() throws IOException { + try { + logScanner.close(); + } catch (Exception e) { + throw new IOException("Failed to close the fluss log scanner", e); + } + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java index 6c8d2ec278bbcd..a34d09439f8cf0 100644 --- a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java @@ -23,47 +23,40 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.table.Table; -import org.apache.fluss.client.table.scanner.ScanRecord; -import org.apache.fluss.client.table.scanner.log.LogScanner; -import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.client.table.scanner.batch.KvSnapshotAndLogBatchScanner; import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; import org.apache.fluss.types.DataType; import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.CloseableIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TimeZone; /** - * Reads one fluss scan range — one bucket of one partition, over a bounded log offset range. + * Reads one fluss scan range — one bucket of one partition, bounded by log offsets. * *

The parameters are the two maps FE built, merged by BE: the scan-level one (connection, table) * and the per-range one (which bucket, which offsets). Nothing between FE and here type-checks them, * so every one this class needs is read through {@link #required}, which names the missing key rather * than letting a null reach fluss. * - *

Where the read stops. A fluss log scanner is a streaming reader with no end; the range's - * stopping offset is what bounds it, and reaching it has to be detected three ways, all of which come - * from fluss's own bounded reader ({@code KvSnapshotAndLogBatchScanner#pollLogRecords}): - *

+ *

Two ways to read, one loop. A log table's range is its bucket's records over + * {@code [start, stop)}, in log order. A primary-key table's range cannot be read that way — its log + * is a change log, so replaying it verbatim returns superseded and deleted rows — and is read instead + * as a kv snapshot merged with the change log that followed it, by fluss's own + * {@code KvSnapshotAndLogBatchScanner}. Both are fluss {@code BatchScanner}s, so the row loop below + * does not know which one it is draining. * *

Partition columns are not read here. FE declares them to the engine, which leaves them out of * {@code required_fields} and fills them from the range itself, so the projection this builds covers @@ -83,12 +76,14 @@ public class FlussJniScanner extends JniScanner { private static final String BUCKET_ID = "fluss.bucket_id"; private static final String LOG_START_OFFSET = "fluss.log_start_offset"; private static final String LOG_STOP_OFFSET = "fluss.log_stop_offset"; + private static final String KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; private static final String RANGE_TYPE_LOG = "LOG"; + private static final String RANGE_TYPE_PK_FULL = "PK_FULL"; /** * How long one poll waits for data. Only affects how often the loop spins, never correctness: the - * loop keeps polling until one of the three stop conditions above fires. + * loop keeps polling until the scanner reports it has reached the end of the range. */ private static final Duration POLL_TIMEOUT = Duration.ofSeconds(1); @@ -96,21 +91,23 @@ public class FlussJniScanner extends JniScanner { private final ClassLoader classLoader; private final FlussColumnValue columnValue; + private final boolean primaryKeyRange; private final long logStopOffset; private final long logStartOffset; + /** Kv snapshot to start a primary-key read from; {@code -1} when the bucket has never had one. */ + private final long kvSnapshotId; private final int bucketId; /** {@code null} on an unpartitioned table, which fluss subscribes to by bucket alone. */ private final Long partitionId; private Connection connection; private Table table; - private LogScanner logScanner; - private TableBucket tableBucket; + private BatchScanner scanner; /** Fluss types of the projected columns, positionally aligned with {@link #fields}. */ private List projectedTypes; - private Iterator pending = Collections.emptyIterator(); + private CloseableIterator currentBatch; private boolean finished; private long rowsRead; @@ -119,17 +116,19 @@ public FlussJniScanner(int batchSize, Map params) { this.classLoader = this.getClass().getClassLoader(); String rangeType = required(RANGE_TYPE); - if (!RANGE_TYPE_LOG.equals(rangeType)) { - // Primary-key and union ranges are planned but not read yet; failing here beats reading - // a primary-key table as a raw changelog and returning superseded rows. + this.primaryKeyRange = RANGE_TYPE_PK_FULL.equals(rangeType); + if (!primaryKeyRange && !RANGE_TYPE_LOG.equals(rangeType)) { + // Union ranges are planned but not read yet; failing here beats returning the fluss half + // of a union read as if it were the whole table. throw new IllegalArgumentException( "fluss scan range type '" + rangeType + "' is not supported yet; expected " - + RANGE_TYPE_LOG); + + RANGE_TYPE_LOG + " or " + RANGE_TYPE_PK_FULL); } // Every range field is parsed here, before open() creates a connection: a range that cannot be // read should say so without having contacted the cluster first. this.logStopOffset = Long.parseLong(required(LOG_STOP_OFFSET)); this.logStartOffset = Long.parseLong(required(LOG_START_OFFSET)); + this.kvSnapshotId = primaryKeyRange ? Long.parseLong(required(KV_SNAPSHOT_ID)) : -1L; this.bucketId = Integer.parseInt(required(BUCKET_ID)); String partition = params.get(PARTITION_ID); this.partitionId = partition == null ? null : Long.parseLong(partition); @@ -167,8 +166,14 @@ public void open() throws IOException { projectedTypes.add(rowType.getTypeAt(index)); } - logScanner = table.newScan().project(scanProjection(projection)).createLogScanner(); - subscribe(); + long tableId = table.getTableInfo().getTableId(); + TableBucket tableBucket = partitionId == null + ? new TableBucket(tableId, bucketId) + : new TableBucket(tableId, partitionId, bucketId); + scanner = primaryKeyRange + ? primaryKeyScanner(tableBucket, projection) + : new BoundedLogBatchScanner(table, tableBucket, scanProjection(projection), + logStartOffset, logStopOffset); } catch (Throwable e) { try { close(); @@ -183,15 +188,23 @@ public void open() throws IOException { } } - private void subscribe() { - long tableId = table.getTableInfo().getTableId(); - if (partitionId == null) { - tableBucket = new TableBucket(tableId, bucketId); - logScanner.subscribe(bucketId, logStartOffset); - } else { - tableBucket = new TableBucket(tableId, partitionId, bucketId); - logScanner.subscribe(partitionId, bucketId, logStartOffset); - } + /** + * The kv snapshot of this bucket merged with the change log that followed it, by fluss's own + * bounded primary-key reader. That class is {@code @Internal} — this connector is pinned to the + * fluss version it ships with, and an upgrade has to re-run these tests. + * + *

The projection goes in as-is, including when it is empty, unlike the log path: this reader + * appends the primary-key columns to what it asks fluss for (it needs them to merge on) and + * projects the merged row back down to exactly what was requested, so it never asks fluss for the + * empty projection fluss rejects. + * + *

It buffers the whole bounded change-log range in memory before merging, which is fluss's own + * design for a batch read; the log tail after a snapshot is what bounds that, so a bucket whose + * snapshot is far behind costs the most. + */ + private BatchScanner primaryKeyScanner(TableBucket tableBucket, int[] projection) { + return new KvSnapshotAndLogBatchScanner( + table, tableBucket, kvSnapshotId, logStartOffset, logStopOffset, projection); } /** @@ -240,33 +253,22 @@ private Configuration clientConfig() { protected int getNext() throws IOException { int rows = 0; while (rows < getBatchSize()) { - if (!pending.hasNext()) { + if (currentBatch == null || !currentBatch.hasNext()) { if (finished) { break; } - pending = poll(); + // A batch scanner returns null only at the end of the range; an empty batch means it + // has more to do first (the primary-key reader drains the change log that way). + currentBatch = scanner.pollBatch(POLL_TIMEOUT); + finished = currentBatch == null; continue; } - ScanRecord record = pending.next(); - if (record.logOffset() >= logStopOffset) { - // Past the end of this range: another query's rows, not ours. - finished = true; - pending = Collections.emptyIterator(); - break; - } - columnValue.setRow(record.getRow()); + columnValue.setRow(currentBatch.next()); for (int i = 0; i < fields.length; i++) { columnValue.setIdx(i, types[i], projectedTypes.get(i)); appendData(i, columnValue); } rows++; - if (record.logOffset() >= logStopOffset - 1) { - // The last record of the range. Do not poll again: the record AT the stopping offset - // may not exist, and waiting for it never returns. - finished = true; - pending = Collections.emptyIterator(); - break; - } } if (fields.length == 0 && rows > 0) { // A count-shaped read projects nothing; the vector table still needs the row count. @@ -276,30 +278,18 @@ protected int getNext() throws IOException { return rows; } - private Iterator poll() { - ScanRecords scanRecords = logScanner.poll(POLL_TIMEOUT); - Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); - if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { - // The fetch reached the end of the range without necessarily yielding a record there — the - // tail can be control records, which take offsets but are never scanned. Without this the - // loop would poll for a row that is never coming. - finished = true; - } - return scanRecords.records(tableBucket).iterator(); - } - @Override public void close() throws IOException { IOException failure = null; // Close everything even if an earlier close throws: a leaked fluss connection keeps its netty // and metadata-updater threads alive for the life of the BE process. - failure = closeQuietly(logScanner, "log scanner", failure); - logScanner = null; + failure = closeQuietly(scanner, "scanner", failure); + scanner = null; failure = closeQuietly(table, "table", failure); table = null; failure = closeQuietly(connection, "connection", failure); connection = null; - pending = Collections.emptyIterator(); + currentBatch = null; if (failure != null) { throw failure; } diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java index 80b0593fe06c16..873b851c167dba 100644 --- a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java @@ -442,15 +442,18 @@ public void missingParameterIsNamed() { Assertions.assertTrue(e.getMessage().contains("fluss.bucket_id"), e.getMessage()); } - /** Primary-key and union ranges are planned by FE but not readable here yet. */ + /** + * Union ranges are planned by FE but not readable here yet. Refusing beats reading their fluss + * half and returning it as if it were the whole table. + */ @Test - public void anUnsupportedRangeTypeIsRefused() { + public void unsupportedRangeTypeIsRefused() { Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); - params.put("fluss.range_type", "PK_FULL"); + params.put("fluss.range_type", "UNION_PK"); IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> new FlussJniScanner(1024, params)); - Assertions.assertTrue(e.getMessage().contains("PK_FULL"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("UNION_PK"), e.getMessage()); } /** A column dropped between planning and reading must say so, not read a neighbouring column. */ diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java new file mode 100644 index 00000000000000..de91df8ed3b7cf --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java @@ -0,0 +1,594 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Reading a fluss primary-key table through the scanner, against a real cluster in this JVM. + * + *

What makes this different from the log test is that the answer is not what was written — it is + * what the writes add up to. A key upserted twice must come back once, with the later value; a deleted + * key must not come back at all; and none of that is visible in the change log the table stores, which + * holds every intermediate state. This is the first line of defence for that merge: a scanner that got + * it wrong would return extra rows that look entirely plausible one at a time. + * + *

Snapshots are triggered, not waited for. The cluster's periodic interval is ten minutes, + * so a bucket has a snapshot only where a test asked for one, which is what makes "these rows came out + * of the snapshot and those out of the log after it" a fact rather than a race. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerPkTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands where the test triggers its snapshot. */ + private static final int BUCKETS = 1; + + /** {@code fluss.kv_snapshot_id} for a bucket fluss has never snapshotted. */ + private static final long NO_SNAPSHOT = -1L; + + /** Fluss's "start at the beginning of the log" sentinel. */ + private static final long EARLIEST = -2L; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_pk_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- the merged view + + /** + * With no snapshot the whole state has to be rebuilt from the change log, and the change log is + * exactly what must not be returned as-is: these five writes produce an insert, an update (which + * fluss logs as two records) and a delete, and the answer is two rows. + */ + @Test + public void changeLogAloneStillYieldsTheMergedView() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_log_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + upsert(tablePath, row(2, "two-updated")); + delete(tablePath, row(3, "three")); + + Object[][] rows = readMerged(tablePath, NO_SNAPSHOT, EARLIEST, + columns("id", "int", "name", "string")); + + Assertions.assertArrayEquals( + new Object[][] {{1, "one"}, {2, "two-updated"}}, sortById(rows)); + } + + /** + * The two halves have to be merged, not concatenated: a key the snapshot holds and the log updates + * must come back once with the log's value, and a key the log deletes must not come back at all + * even though the snapshot still has it. + */ + @Test + public void logAfterASnapshotOverridesAndDeletesWhatTheSnapshotHolds() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_snapshot_and_log"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + long snapshotId = snapshot(tablePath); + upsert(tablePath, row(2, "two-after-snapshot"), row(4, "four")); + delete(tablePath, row(3, "three")); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), + columns("id", "int", "name", "string")); + + Assertions.assertArrayEquals(new Object[][] { + {1, "one"}, {2, "two-after-snapshot"}, {4, "four"}}, sortById(rows)); + } + + /** Nothing written since the snapshot: the snapshot alone is the whole answer. */ + @Test + public void snapshotWithNoLogTailIsReadOnItsOwn() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_snapshot_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long snapshotId = snapshot(tablePath); + long logOffset = snapshotLogOffset(tablePath); + + // Start and stop meet, which is what planning produces for a bucket whose log has caught up. + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + snapshotId, logOffset, logOffset), 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "one"}, {2, "two"}}, sortById(rows)); + } + + /** + * Rows are compared to the state at the stopping offset, not to the state now. A write that lands + * after planning belongs to the next query; letting it through would make two scans of the same + * plan disagree. + */ + @Test + public void writesAfterTheStoppingOffsetAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_late_write"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long stopOffset = latestOffset(tablePath); + upsert(tablePath, row(2, "two-later"), row(3, "three")); + + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + NO_SNAPSHOT, EARLIEST, stopOffset), 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "one"}, {2, "two"}}, sortById(rows)); + } + + // ---------------------------------------------------------------- values + + /** + * Every mapped type, half of it read out of a kv snapshot and half out of the change log after it. + * The two are stored in different row formats, and a value wrapper that only handles one of them + * would still pass a test that read a single row from a single place. + */ + @Test + public void everyMappedTypeSurvivesBothHalvesOfTheMerge() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_all_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("c_boolean", DataTypes.BOOLEAN()) + .column("c_tinyint", DataTypes.TINYINT()) + .column("c_smallint", DataTypes.SMALLINT()) + .column("c_bigint", DataTypes.BIGINT()) + .column("c_float", DataTypes.FLOAT()) + .column("c_double", DataTypes.DOUBLE()) + .column("c_char", DataTypes.CHAR(5)) + .column("c_string", DataTypes.STRING()) + .column("c_decimal", DataTypes.DECIMAL(20, 4)) + .column("c_date", DataTypes.DATE()) + .column("c_timestamp", DataTypes.TIMESTAMP(6)) + .column("c_timestamp_ltz", DataTypes.TIMESTAMP_LTZ(6)) + .column("c_array", DataTypes.ARRAY(DataTypes.INT())) + .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("c_struct", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT()), + DataTypes.FIELD("b", DataTypes.STRING()))) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + + LocalDateTime timestamp = LocalDateTime.of(2026, 8, 3, 12, 34, 56, 123456000); + Instant instant = Instant.parse("2026-08-03T04:34:56.123456Z"); + Map map = new LinkedHashMap<>(); + map.put(BinaryString.fromString("k1"), 1); + upsert(tablePath, GenericRow.of(1, true, (byte) 1, (short) 2, 4L, 5.5f, 6.5d, + BinaryString.fromString("abcde"), BinaryString.fromString("hello"), + Decimal.fromBigDecimal(new BigDecimal("12345.6789"), 20, 4), + (int) LocalDate.of(2026, 8, 3).toEpochDay(), + TimestampNtz.fromLocalDateTime(timestamp), + TimestampLtz.fromInstant(instant), + new GenericArray(new int[] {10, 20}), new GenericMap(map), + GenericRow.of(7, BinaryString.fromString("inner")))); + long snapshotId = snapshot(tablePath); + // Row 2 goes only into the log after the snapshot, and is all-null besides its key: a column + // read through the wrong accessor often produces a plausible value for real data and only + // misbehaves on null. + upsert(tablePath, GenericRow.of(2, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null)); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), columns( + "id", "int", + "c_boolean", "boolean", + "c_tinyint", "tinyint", + "c_smallint", "smallint", + "c_bigint", "bigint", + "c_float", "float", + "c_double", "double", + "c_char", "char(5)", + "c_string", "string", + "c_decimal", "decimal(20,4)", + "c_date", "datev2", + "c_timestamp", "datetimev2(6)", + "c_timestamp_ltz", "timestamptz(6)", + "c_array", "array", + "c_map", "map", + "c_struct", "struct")); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(2, sorted.length); + Object[] fromSnapshot = sorted[0]; + Assertions.assertEquals(1, fromSnapshot[0]); + Assertions.assertEquals(true, fromSnapshot[1]); + Assertions.assertEquals((byte) 1, fromSnapshot[2]); + Assertions.assertEquals((short) 2, fromSnapshot[3]); + Assertions.assertEquals(4L, fromSnapshot[4]); + Assertions.assertEquals(5.5f, fromSnapshot[5]); + Assertions.assertEquals(6.5d, fromSnapshot[6]); + Assertions.assertEquals("abcde", fromSnapshot[7]); + Assertions.assertEquals("hello", fromSnapshot[8]); + Assertions.assertEquals(new BigDecimal("12345.6789"), fromSnapshot[9]); + Assertions.assertEquals(LocalDate.of(2026, 8, 3), fromSnapshot[10]); + Assertions.assertEquals(timestamp, fromSnapshot[11]); + // TIMESTAMPTZ is carried as the UTC wall clock of the instant, not the session zone's. + Assertions.assertEquals(LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC), + fromSnapshot[12]); + Assertions.assertEquals("[10, 20]", String.valueOf(fromSnapshot[13])); + Assertions.assertEquals("{k1=1}", String.valueOf(fromSnapshot[14])); + Assertions.assertEquals("{a=7, b=inner}", String.valueOf(fromSnapshot[15])); + + Object[] fromLog = sorted[1]; + Assertions.assertEquals(2, fromLog[0]); + for (int i = 1; i < fromLog.length; i++) { + Assertions.assertNull(fromLog[i], "column " + i + " of the all-null row came back non-null"); + } + } + + // ---------------------------------------------------------------- projection and shape + + /** + * The merge needs the primary key even when the query does not ask for it, so the reader adds it + * to what it fetches and projects it back out. Getting that wrong shows up as an extra column, or + * as columns in the fetch order rather than the requested one. + */ + @Test + public void projectionExcludingThePrimaryKeyKeepsTheRequestedShape() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("a", DataTypes.INT()) + .column("b", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + upsert(tablePath, GenericRow.of(1, 10, BinaryString.fromString("x"))); + long snapshotId = snapshot(tablePath); + upsert(tablePath, GenericRow.of(1, 11, BinaryString.fromString("y"))); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), + columns("b", "string", "a", "int")); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {"y", 11}, rows[0]); + } + + /** + * A query can need no column of this scanner at all — {@code select dt, count(*) ... group by dt} + * over a partitioned table — and the row count still has to be the merged one. Counting change log + * records instead would report four rows where the table holds two. + */ + @Test + public void projectingNoColumnStillCountsTheMergedRows() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_count_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(3, "three")); + + Map params = pkParams(tablePath, columns(), NO_SNAPSHOT, EARLIEST, + latestOffset(tablePath)); + int rows = 0; + FlussJniScanner scanner = new FlussJniScanner(1024, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + rows += scanner.getTable().getNumRows(); + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + + Assertions.assertEquals(2, rows); + } + + /** More merged rows than fit in one batch: the reader has to resume where it stopped. */ + @Test + public void mergedRowsSpanSeveralBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_batches"); + createPkTable(tablePath); + GenericRow[] first = new GenericRow[50]; + for (int i = 0; i < 50; i++) { + first[i] = row(i, "v" + i); + } + upsert(tablePath, first); + long snapshotId = snapshot(tablePath); + GenericRow[] second = new GenericRow[50]; + for (int i = 50; i < 100; i++) { + second[i - 50] = row(i, "v" + i); + } + upsert(tablePath, second); + + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + snapshotId, snapshotLogOffset(tablePath), latestOffset(tablePath)), 7); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(100, sorted.length); + for (int i = 0; i < 100; i++) { + Assertions.assertArrayEquals(new Object[] {i, "v" + i}, sorted[i], "row " + i); + } + } + + /** + * A partitioned primary-key table is snapshotted per partition, and the scanner has to subscribe to + * the partition's bucket rather than the table's. Subscribing without the partition reads a + * different tablet altogether. + */ + @Test + public void partitionedTableIsReadThroughItsPartitionBucket() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_part"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), BinaryString.fromString("a")), + GenericRow.of(2, BinaryString.fromString("2026_08_04"), BinaryString.fromString("b"))); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), partitionId, 0)); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), + BinaryString.fromString("a-updated"))); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath, "2026_08_03").get(); + long stop = admin.listOffsets(tablePath, "2026_08_03", Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + // The partition column is not read here: FE declares it and BE materializes it from the range. + Map params = pkParams(tablePath, columns("id", "int", "name", "string"), + snapshots.getSnapshotId(0).orElse(NO_SNAPSHOT), + snapshots.getLogOffset(0).orElse(EARLIEST), stop); + params.put("fluss.partition_id", String.valueOf(partitionId)); + + Object[][] rows = read(params, 1024); + + // Only this partition's row, and with the update applied. + Assertions.assertArrayEquals(new Object[][] {{1, "a-updated"}}, sortById(rows)); + } + + /** A snapshot id pointing at a snapshot that does not exist must fail, not read an empty table. */ + @Test + public void unknownSnapshotIdFailsRatherThanReadingNothing() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_bad_snapshot"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + + Map params = pkParams(tablePath, columns("id", "int", "name", "string"), + 999L, 0L, latestOffset(tablePath)); + + Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + } + + // ---------------------------------------------------------------- helpers + + private void createPkTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + } + + private static GenericRow row(int id, String name) { + return GenericRow.of(id, BinaryString.fromString(name)); + } + + private static void upsert(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (GenericRow row : rows) { + writer.upsert(row); + } + writer.flush(); + } + } + + private static void delete(TablePath tablePath, GenericRow row) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + writer.delete(row); + writer.flush(); + } + } + + /** Takes a kv snapshot of bucket 0 and returns its id. */ + private static long snapshot(TablePath tablePath) throws Exception { + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), 0)); + return admin.getLatestKvSnapshots(tablePath).get().getSnapshotId(0) + .orElseThrow(() -> new AssertionError("the trigger produced no snapshot")); + } + + private static long snapshotLogOffset(TablePath tablePath) throws Exception { + return admin.getLatestKvSnapshots(tablePath).get().getLogOffset(0) + .orElseThrow(() -> new AssertionError("the bucket has no snapshot")); + } + + private static long latestOffset(TablePath tablePath) throws Exception { + return admin.listOffsets(tablePath, Collections.singletonList(0), new OffsetSpec.LatestSpec()) + .all().get().get(0); + } + + private static long tableId(TablePath tablePath) throws Exception { + return admin.getTableInfo(tablePath).get().getTableId(); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + /** Reads the whole bucket up to where its log has got to, the way a planned range would. */ + private static Object[][] readMerged(TablePath tablePath, long snapshotId, long logStartOffset, + Map columns) throws Exception { + return read(pkParams(tablePath, columns, snapshotId, logStartOffset, latestOffset(tablePath)), + 1024); + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** The merged map BE hands the scanner for a primary-key range. */ + private static Map pkParams(TablePath tablePath, Map columns, + long snapshotId, long logStartOffset, long logStopOffset) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "PK_FULL"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.kv_snapshot_id", String.valueOf(snapshotId)); + params.put("fluss.log_start_offset", String.valueOf(logStartOffset)); + params.put("fluss.log_stop_offset", String.valueOf(logStopOffset)); + params.put("time_zone", "UTC"); + return params; + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced. {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; + * reading it as rows would silently compare a column against a row. + */ + private static Object[][] read(Map params, int batchSize) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } + + /** + * Sorted by the first column, which every fixture here makes the primary key. The merge returns + * rows in primary-key encoding order, which is not the natural order of the key's Java type, so + * asserting the reader's order would be asserting fluss's key encoding. + */ + private static Object[][] sortById(Object[][] rows) { + Object[][] sorted = Arrays.copyOf(rows, rows.length); + Arrays.sort(sorted, Comparator.comparingInt(row -> (Integer) row[0])); + return sorted; + } +} From 98d3d7e806775df8fb842d5ace843efea8d8b6fc Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 07:46:28 +0800 Subject: [PATCH 14/35] [test](fluss) Bake a kv snapshot into the e2e fixtures and read them back A fluss primary-key table with no kv snapshot is read by replaying its whole change log, which is correct and takes a completely different code path from the one this environment exists to cover: BE reads snapshot FILES that the fluss container wrote, from the host, at an absolute path both sides share. With the default ten-minute interval whether a suite exercised that path was down to when it happened to run. The server now snapshots every ten seconds and startup does not report the environment ready until every primary-key fixture has one on disk. That does not pile up files -- a tablet whose log has not advanced since its last snapshot is skipped -- and the wait proves a snapshot was taken, not that the coordinator has committed it (completion is registered in ZooKeeper and the directory here is created before the upload). Losing that millisecond costs a suite nothing: the read falls back to the change log and still returns the right rows. Adds pk_part, a partitioned primary-key table with an update in one partition and a delete in another: a partitioned table is snapshotted per partition, so a merge that crossed partitions loses or resurrects one of them. KNOWN FAILING, and the reason is not in this commit: reading a kv snapshot inside BE aborts the process. The RocksDB JNI library that fluss-client bundles and doris_be define 2576 rocksdb symbols under identical mangled names, and the executable wins the lookup, so the library's calls land in a different RocksDB built against a different libstdc++ string ABI. See plan-doc/HANDOFF.md. The same scanner code passes 10/10 against a real cluster in a plain JVM, which is the A/B. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 18 ++ .../docker-compose/fluss/fluss.yaml.tpl | 13 + .../fluss/scripts/run-init-sql.sh | 59 ++++- .../docker-compose/fluss/sql/init.sql | 30 +++ .../fluss/test_fluss_catalog.groovy | 3 +- .../fluss/test_fluss_pk_table.groovy | 226 ++++++++++++++++++ 6 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index 3389655713f874..9169f0f6d4c603 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -85,5 +85,23 @@ path string. | `log_empty` | log table with no rows at all (planning must emit zero scan ranges) | | `pk_basic` | primary-key table, one updated row and one deleted row | | `pk_types` | primary-key table with the same type coverage as `log_types` | +| `pk_part` | primary-key table partitioned by `dt`, with an update and a delete inside a partition | Data-lake tables and the tiering service are added when union read lands. + +### Primary-key tables come with a kv snapshot + +The server takes kv snapshots every ten seconds here rather than every ten +minutes, and startup does not report the environment ready until each +primary-key table has one on disk (`wait_for_kv_snapshots` in +`scripts/run-init-sql.sh`). + +That is not tuning: Doris BE reads those snapshot files directly, from the +host, at the path this container wrote them to — the directory is bind mounted +at the same absolute path on both sides — and nothing but an end-to-end run +covers that. A primary-key table with no snapshot is read by replaying its +whole change log instead, which is equally correct and takes a different code +path, so without the wait the interesting path would only be exercised by luck. + +Short intervals do not pile up files: a tablet whose log has not advanced since +its last snapshot is skipped, and the fixtures stop writing when init ends. diff --git a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl index f9b419e69527b8..e071113fd132c2 100644 --- a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl +++ b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl @@ -98,6 +98,15 @@ services: # rejects every write, and the client retries ~2^31 times instead of # failing, so the environment hangs instead of reporting anything. server.data-disk.write-limit-ratio: 1.0 + # Ten minutes by default, which would leave the primary-key fixtures + # with no kv snapshot for as long as the suites take to run: they would + # then be read by replaying the change log, and the path where Doris BE + # reads a snapshot FILE this container wrote -- the one thing only an + # end-to-end run can check -- would never be exercised. Short enough + # that init can wait for it; this does not pile up files, because a + # tablet whose log has not advanced since its last snapshot is skipped + # (KvTabletSnapshotTarget), and the fixtures stop writing after init. + kv.snapshot.interval: 10s volumes: - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} healthcheck: @@ -172,6 +181,9 @@ services: environment: - FLUSS_BOOTSTRAP_SERVERS=doris--fluss-coordinator:9123 - FLUSS_JOBMANAGER_HOST=doris--fluss-jobmanager + # Read-only, and only so that init can wait for the kv snapshots to be + # written before declaring the environment ready. + - FLUSS_REMOTE_DATA_DIR=${FLUSS_REMOTE_DATA_DIR} - | FLINK_PROPERTIES= jobmanager.rpc.address: doris--fluss-jobmanager @@ -179,6 +191,7 @@ services: volumes: - ./sql:/opt/fluss-sql:ro - ./scripts:/opt/fluss-scripts:ro + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro healthcheck: test: ["CMD-SHELL", "test -f /tmp/fluss-init/SUCCESS"] interval: 5s diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh index 0709b870e6e864..e1d783143c6084 100755 --- a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -31,6 +31,10 @@ JOBMANAGER_PORT=8081 WAIT_SECONDS=180 SQL_TIMEOUT_SECONDS=900 ATTEMPTS=3 +# Primary-key fixtures whose buckets must have been snapshotted before the +# environment counts as ready. See wait_for_kv_snapshots. +SNAPSHOT_TABLES=(pk_basic pk_types pk_part) +SNAPSHOT_WAIT_SECONDS=120 rm -rf "${MARKER_DIR}" mkdir -p "${MARKER_DIR}" @@ -54,6 +58,55 @@ wait_for_jobmanager sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ "${SQL_TEMPLATE}" >"${MARKER_DIR}/init.sql" +# Waits until every primary-key fixture has a kv snapshot on disk. +# +# Doris BE reads those files directly, from the host, at the path this container +# wrote them to -- a bind mount at the same absolute path on both sides. Nothing +# but an end-to-end run covers that, and without this wait it would only be +# covered by luck: with no snapshot a primary-key table is read by replaying its +# whole change log, which is equally correct and takes a different code path +# entirely. Baking the snapshot into the environment makes every later suite +# exercise the file-reading path instead of racing the ten-second interval. +# +# What it proves is that a snapshot was taken, not that the coordinator has +# committed it -- completion is registered in ZooKeeper, and the directory here +# is created before the upload. That gap is milliseconds, and losing it costs a +# suite nothing: the read falls back to the change log and still returns the +# right rows. Never seeing a snapshot at all is the real problem, and that is +# what the timeout reports. +wait_for_kv_snapshots() { + local waited=0 + local table + local missing + while :; do + missing="" + for table in "${SNAPSHOT_TABLES[@]}"; do + # Two shapes, because a partition sits between the table and the + # bucket ({partitionName}-p{partitionId}); the trailing /* requires + # the snapshot directory to hold a file, not merely to exist. + # {remote.data.dir}/kv/{db}/{table}-{tableId}/{bucket}/snap-{id}/ + # {remote.data.dir}/kv/{db}/{table}-{tableId}/{partition}/{bucket}/snap-{id}/ + local root="${FLUSS_REMOTE_DATA_DIR}/kv/fluss_test/${table}-" + if ! compgen -G "${root}*/*/snap-*/*" >/dev/null 2>&1 \ + && ! compgen -G "${root}*/*/*/snap-*/*" >/dev/null 2>&1; then + missing="${missing} ${table}" + fi + done + if [[ -z "${missing}" ]]; then + echo "Kv snapshots present for:${SNAPSHOT_TABLES[*]}" + return 0 + fi + if ((waited >= SNAPSHOT_WAIT_SECONDS)); then + echo "ERROR: no kv snapshot after ${SNAPSHOT_WAIT_SECONDS}s for:${missing}" >&2 + echo "ERROR: expected under ${FLUSS_REMOTE_DATA_DIR}/kv/fluss_test/" >&2 + ls -R "${FLUSS_REMOTE_DATA_DIR}/kv" >&2 2>/dev/null || true + return 1 + fi + sleep 5 + waited=$((waited + 5)) + done +} + run_attempt() { local log="$1" local status=0 @@ -79,8 +132,12 @@ run_attempt() { for ((attempt = 1; attempt <= ATTEMPTS; attempt++)); do echo "Running fluss init SQL (attempt ${attempt}/${ATTEMPTS})" if run_attempt "${MARKER_DIR}/init-attempt-${attempt}.log"; then + echo "Fluss init SQL finished; waiting for kv snapshots" + if ! wait_for_kv_snapshots; then + exit 1 + fi touch "${MARKER_DIR}/SUCCESS" - echo "Fluss init SQL finished" + echo "Fluss environment ready" exec tail -f /dev/null fi echo "Fluss init SQL failed on attempt ${attempt}" >&2 diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index 41d1274971079a..34f8a1f6d9babe 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -277,3 +277,33 @@ INSERT INTO pk_types VALUES CAST(NULL AS MAP), CAST(NULL AS ROW) ); + +-- --------------------------------------------------------------------------- +-- pk_part: partitioned primary-key table. A partitioned primary-key table is +-- snapshotted per partition, so reading it wrong -- asking the table for its +-- snapshots instead of the partition -- resumes the change log at another +-- partition's offset. One row of one partition is updated and one deleted, so +-- the merge has to happen inside a partition and not across the table. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_part ( + id INT NOT NULL, + name STRING, + dt STRING NOT NULL, + PRIMARY KEY (id, dt) NOT ENFORCED +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO pk_part VALUES + (1, 'q1a', '20260101'), + (2, 'q1b', '20260101'), + (3, 'q2a', '20260102'), + (4, 'q2b', '20260102'); + +INSERT INTO pk_part VALUES + (2, 'q1b-updated', '20260101'); + +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM pk_part WHERE id = 4 AND dt = '20260102'; +SET 'execution.runtime-mode' = 'streaming'; diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy index 3270bd77361115..2f19c9b64937fa 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -51,7 +51,8 @@ suite("test_fluss_catalog", "p0,external") { def tableRows = sql """show tables""" def tables = tableRows.collect { it[0] } - for (String expected : ["log_basic", "log_types", "log_part", "log_empty", "pk_basic", "pk_types"]) { + for (String expected : ["log_basic", "log_types", "log_part", "log_empty", "pk_basic", "pk_types", + "pk_part"]) { assertTrue(tables.contains(expected), "table ${expected} missing: ${tables}") } diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy new file mode 100644 index 00000000000000..1ca54f442bab52 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading fluss primary-key tables end to end. What separates this from the log +// suite is that the answer is not what the fixture wrote but what the writes add +// up to: a fluss primary-key table stores a change log holding every intermediate +// state, and the read has to merge a kv snapshot with the log that followed it. +// Returning the change log instead would show up here as extra rows that each look +// entirely plausible. +// +// The part only this suite can cover is where those snapshot files come from: the +// fluss container writes them and Doris BE reads them from the host filesystem, at +// the same absolute path on both sides. Startup does not report the environment +// ready until every primary-key fixture has a snapshot on disk (see the docker +// README), so these queries take that path rather than replaying the change log. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are +// static - this suite never writes, so no polling gate is needed. Everything is +// asserted explicitly rather than through qt_ and a .out file: the values are the +// fixture's own literals, so an expectation that drifts from the fixture is a diff +// in one file rather than a regenerated baseline nobody reads. +suite("test_fluss_pk_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_pk_table" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only; the legacy one answers + // "Not supported create reader for table format: fluss". Fuzzy sessions randomize + // this variable, so pinning it is what keeps the suite from failing on half the + // CI runs for a reason that has nothing to do with fluss. + sql """set enable_file_scanner_v2 = true""" + + def scalarOf = { String query -> sql(query)[0][0].toString() } + def planOf = { String query -> + def planRows = sql("""explain ${query}""") + return planRows.collect { it[0].toString() }.join("\n") + } + def pkRangesOf = { String plan -> + def matcher = (plan =~ /pkRanges=(\d+)/) + assertTrue(matcher.find(), "plan has no flussScan line: ${plan}") + return matcher.group(1) as int + } + + // --- the merged view ---------------------------------------------------- + // The fixture inserts four rows, updates one and deletes one. Four writes, three + // rows: id 2 carries its later value and id 3 is gone. Reading the change log + // straight through would answer six rows here, all of them real records. + def basicRows = sql """select id, name, score from pk_basic order by id""" + assertEquals(3, basicRows.size()) + assertEquals(["1", "k1", "1.5"], basicRows[0].collect { it.toString() }) + assertEquals(["2", "k2-updated", "22.5"], basicRows[1].collect { it.toString() }) + assertEquals(["4", "k4", "4.5"], basicRows[2].collect { it.toString() }) + + // COUNT(*) projects no column at all. On a primary-key table the count still has + // to be the merged one: counting change log records would report six. + assertEquals("3", scalarOf("""select count(*) from pk_basic""")) + + // The deleted key must be absent, not merely superseded. + assertEquals("0", scalarOf("""select count(*) from pk_basic where id = 3""")) + // And the updated key must appear once, not once per version. + assertEquals("1", scalarOf("""select count(*) from pk_basic where id = 2""")) + assertEquals("0", scalarOf("""select count(*) from pk_basic where name = 'k2'""")) + + // --- projection and predicates ----------------------------------------- + // The merge needs the primary key even when the query does not ask for it, so the + // reader adds it to what it fetches and projects it back out. A leaked key column + // or a fetch-order projection shows up as the wrong values here. + def names = sql """select name from pk_basic order by name""" + assertEquals(["k1", "k2-updated", "k4"], names.collect { it[0].toString() }) + + def reordered = sql """select score, name from pk_basic where id = 1""" + assertEquals(["1.5", "k1"], reordered[0].collect { it.toString() }) + + assertEquals("2", scalarOf("""select count(*) from pk_basic where score > 2.0""")) + + // --- every mapped type, in the kv row format --------------------------- + // Primary-key tables store rows in a different format from a log table's, so this + // repeats the type coverage rather than trusting the log suite for it. Asserted by + // predicate, not by rendering: how a decimal or a map prints is the display layer's + // business, and pinning it here would make this suite fail for the wrong reasons. + assertEquals("2", scalarOf("""select count(*) from pk_types""")) + + assertEquals("1", scalarOf(""" + select count(*) from pk_types where id = 1 + and f_boolean = true + and f_tinyint = 1 + and f_smallint = 2 + and f_int = 3 + and f_bigint = 4 + and f_float = cast(1.5 as float) + and f_double = 2.5 + and f_decimal = 123.4567 + """)) + + // Fluss BYTES and BINARY map to a Doris string by default, so their content is + // compared as hex; a decoder that lost the length would return a prefix of this. + assertEquals("1", scalarOf(""" + select count(*) from pk_types where id = 1 + and f_char = 'char1' + and f_string = 'string1' + and hex(f_binary) = '010203' + and hex(f_bytes) = '0A0B' + """)) + + // TIMESTAMP_LTZ is only checked for presence: its rendering depends on the session + // time zone, which is not what this suite is pinning. + assertEquals("1", scalarOf(""" + select count(*) from pk_types where id = 1 + and f_date = '2026-01-01' + and f_timestamp = '2026-01-01 01:02:03.456789' + and f_timestamp_ltz is not null + """)) + + assertEquals("1", scalarOf(""" + select count(*) from pk_types where id = 1 + and array_size(f_array) = 3 + and f_array[1] = 1 + and f_array[3] = 3 + and f_map['k1'] = 1 + and f_map['k2'] = 2 + and struct_element(f_row, 'r_int') = 1 + and struct_element(f_row, 'r_string') = 'nested1' + """)) + + // The all-NULL row. Read through a null map that is off by one column, every value + // after it shifts, so this is checked column by column rather than by row count. + assertEquals("1", scalarOf(""" + select count(*) from pk_types where id = 2 + and f_boolean is null and f_tinyint is null and f_smallint is null + and f_int is null and f_bigint is null and f_float is null + and f_double is null and f_decimal is null and f_char is null + and f_string is null and f_binary is null and f_bytes is null + and f_date is null and f_timestamp is null and f_timestamp_ltz is null + and f_array is null and f_map is null and f_row is null + """)) + + // --- partitioned primary-key table -------------------------------------- + // A partitioned primary-key table is snapshotted per partition, so its snapshots + // have to be asked for per partition too; asking at table level resumes the change + // log at another partition's offset. The partition column itself is not read by the + // scanner - FE declares it and BE fills it in from each range - so checking it + // against the row it belongs to is what catches a partition value on the wrong split. + def partRows = sql """select id, name, dt from pk_part order by dt, id""" + assertEquals(3, partRows.size()) + assertEquals(["1", "q1a", "20260101"], partRows[0].collect { it.toString() }) + assertEquals(["2", "q1b-updated", "20260101"], partRows[1].collect { it.toString() }) + assertEquals(["3", "q2a", "20260102"], partRows[2].collect { it.toString() }) + + // The delete landed in 20260102 and the update in 20260101: a merge that crossed + // partitions would lose or resurrect one of them. + def perPartition = sql """select dt, count(*) from pk_part group by dt order by dt""" + assertEquals(2, perPartition.size()) + assertEquals(["20260101", "2"], perPartition[0].collect { it.toString() }) + assertEquals(["20260102", "1"], perPartition[1].collect { it.toString() }) + + def prunedRows = sql """select id, name from pk_part where dt = '20260101' order by id""" + assertEquals(2, prunedRows.size()) + assertEquals(["1", "q1a"], prunedRows[0].collect { it.toString() }) + assertEquals(["2", "q1b-updated"], prunedRows[1].collect { it.toString() }) + + // --- planning is visible in the plan ------------------------------------ + def basicPlan = planOf("""select * from pk_basic""") + assertTrue(basicPlan.contains("flussScan: unionRead=no"), + "no fluss scan line: ${basicPlan}") + assertTrue(basicPlan.contains("lakeSplits=0"), "unexpected lake splits: ${basicPlan}") + assertTrue(basicPlan.contains("mode=auto"), "unexpected union read mode: ${basicPlan}") + // A primary-key table produces primary-key ranges and no log ranges. Counting them + // together would hide a table planned as a raw log read, which returns superseded + // rows and otherwise looks like a working query. + assertTrue(basicPlan.contains("logRanges=0"), + "a primary-key table was planned as a log read: ${basicPlan}") + // One range per bucket that holds anything. Which of the three buckets a key hashes + // into is fluss's choice, so the count is bounded rather than fixed. + def basicRanges = pkRangesOf(basicPlan) + assertTrue(basicRanges >= 1 && basicRanges <= 3, + "pk_basic planned ${basicRanges} ranges over 3 buckets") + + def fullPartPlan = planOf("""select * from pk_part""") + assertTrue(fullPartPlan.contains("partition=2/2"), + "both partitions should be scanned: ${fullPartPlan}") + + def prunedPlan = planOf("""select * from pk_part where dt = '20260101'""") + assertTrue(prunedPlan.contains("partition=1/2"), + "partition pruning did not reach the connector: ${prunedPlan}") + // Pruning has to shrink the work, not just the plan line: a partition name rendered + // one way for the listing and another way for the match would prune to nothing here + // while still reporting 1/2. + def prunedRanges = pkRangesOf(prunedPlan) + assertTrue(prunedRanges >= 1 && prunedRanges <= 2, + "one partition has 2 buckets, planned ${prunedRanges} ranges: ${prunedPlan}") + assertTrue(prunedRanges <= pkRangesOf(fullPartPlan), + "pruning planned more ranges than the full scan: ${prunedPlan}") + + sql """switch internal""" + sql """drop catalog ${catalogName}""" +} From 4b9d66606cc488c5b668448a8e3ed063e713a595 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 09:04:43 +0800 Subject: [PATCH 15/35] [fix](be) Stop exporting the statically linked RocksDB symbols Exporting them makes this executable the definition every later-loaded library binds to, so a JNI library carrying its own RocksDB runs half on ours. The fluss scanner bundles frocksdbjni, whose librocksdbjni.so defines 2576 rocksdb symbols under names identical to ours but was built against the pre-C++11 libstdc++ string ABI: objects laid out by one copy and used by the other yield a garbage length, an std::bad_alloc that escapes the JNI frame, and an aborted BE. Reading any fluss primary-key table with a kv snapshot killed the process, reproducibly. Scoped to the archive rather than dropping ENABLE_EXPORTS, because what needs the exports is native UDFs (runtime/user_function_cache.cpp dlopens them) and those use the Doris UDF ABI, which has nothing to do with RocksDB. Crash stacks do not need it either -- they are symbolized from debug info, which is why they name even anonymous-namespace functions. 61 rocksdb symbols remain exported: inline and template members the compiler emitted into Doris's own objects, which no archive exclusion can reach. 29 of those still share a name with the JNI library, but none appear in its relocation table -- it never resolves them at load time, so they cannot be interposed. The library also duplicates zstd, lz4, snappy, bzip2 and zlib symbols; those are C ABIs, stable and layout-free, and are left alone. Verified: the fluss primary-key suite passes with BE alive (it aborted before); all three fluss suites green; an internal table survives write, BE restart and read, which is the tablet metadata RocksDB itself round-tripping. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- be/src/service/CMakeLists.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index b9bcef0b7b821a..64ed8651417748 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -49,6 +49,24 @@ if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF") # This permits libraries loaded by dlopen to link to the symbols in the program. set_target_properties(doris_be PROPERTIES ENABLE_EXPORTS 1) + # ...but not the symbols of the RocksDB we link statically. Exporting those makes this + # executable the definition every later-loaded library binds to, and a JNI library that + # carries its own RocksDB then runs half on ours: the fluss scanner bundles frocksdbjni, + # whose librocksdbjni.so defines 2576 rocksdb symbols under names identical to ours but + # was built against the pre-C++11 libstdc++ string ABI. Objects laid out by one and used + # by the other yield a garbage length, an std::bad_alloc that escapes the JNI frame, and + # an aborted BE. Hiding this archive lets that library bind to its own copy. + # + # Scoped to the archive rather than dropping ENABLE_EXPORTS: what needs the exports is + # native UDFs (runtime/user_function_cache.cpp dlopens them), and those use the Doris UDF + # ABI, which has nothing to do with RocksDB. Crash stacks do not need it either -- they are + # symbolized from debug info, which is why they name even anonymous-namespace functions. + # + # The same library also duplicates zstd, lz4, snappy, bzip2 and zlib symbols. Those are C + # ABIs, stable across versions and layout-free, so they are left alone until something + # shows otherwise -- unlike RocksDB, whose C++ objects are what actually corrupt. + target_link_options(doris_be PRIVATE "-Wl,--exclude-libs,librocksdb.a") + target_link_libraries(doris_be ${DORIS_LINK_LIBS} ) From 6a5d5d4212b8466ffc8e31f0510d2404d1576fd2 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 10:06:00 +0800 Subject: [PATCH 16/35] [feat](fluss) Read a fluss lake table through the paimon connector A datalake-enabled fluss table now exposes a "lake" system table, so tbl$lake reads the paimon table its tiering service writes. The read is delegated whole: the handle comes from an embedded paimon sibling connector, the engine routes that handle's scan to the sibling's own planner, and this connector's metadata forwards every per-handle call. The sibling is built through ConnectorContext.createSiblingConnector, so the fluss plugin bundles no paimon class at all -- the same contract the hive gateway uses for its iceberg and hudi tables, and the reason its zip is unchanged at four artifacts. Its catalog properties come from the fluss TABLE's properties, where the coordinator injects the cluster's datalake.paimon.* settings; only a filesystem catalog is served for now, and anything else fails loud rather than half-configuring a catalog. Exactly one sibling per catalog, which is a correctness constraint and not thrift: two paimon siblings answer "is this handle yours?" with the same class test, so a second one could never be routed apart from the first and a table would silently read the wrong warehouse. A second lake configuration is refused with a message asking for a catalog refresh. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../doris/connector/fluss/FlussConnector.java | 137 +++++++- .../fluss/FlussConnectorMetadata.java | 164 ++++++++- .../fluss/PaimonSiblingProperties.java | 111 ++++++ .../fluss/FlussConnectorLakeSiblingTest.java | 182 ++++++++++ .../fluss/FlussConnectorMetadataTest.java | 18 +- .../connector/fluss/FlussLakeTableTest.java | 319 ++++++++++++++++++ .../connector/fluss/FlussSplitPlanTest.java | 6 +- .../fluss/PaimonSiblingPropertiesTest.java | 157 +++++++++ .../connector/fluss/RecordingLakeSibling.java | 180 ++++++++++ 9 files changed, 1263 insertions(+), 11 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 82b39712ecd5b2..481fe54edd4ca4 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -22,6 +22,7 @@ import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorTestResult; import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.spi.ConnectorContext; @@ -45,21 +46,49 @@ */ public class FlussConnector implements Connector { + /** + * The sibling connector type a fluss lake table ({@code tbl$lake}) is delegated to. A string literal, + * not the paimon plugin's own constant: that plugin loads child-first and none of its classes are + * visible from here. Matches the type {@code PaimonConnectorProvider} registers. + */ + private static final String PAIMON_CONNECTOR_TYPE = "paimon"; + private final Map properties; private final String catalogName; + private final ConnectorContext context; private volatile Connection connection; + // The embedded paimon SIBLING connector this catalog delegates its lake tables to. Built lazily in the + // PAIMON plugin's OWN child-first classloader via context.createSiblingConnector, never co-packaged + // into the fluss zip (a second copy of paimon in one JVM). Held ONLY as the parent-first Connector + // interface and NEVER cast: the concrete type is invisible to the fluss loader, so a cast would CCE + // across the loader split. + // + // Exactly ONE, and that is a correctness constraint rather than thrift. Handle routing asks each + // sibling "is this handle yours?", and two paimon siblings answer that question with the SAME class + // test — so a second one could never be told apart from the first and a table would silently read the + // wrong warehouse. One fluss cluster injects one lake configuration into all of its datalake tables + // (LakeCatalogDynamicLoader), so one is also all a healthy catalog ever needs; a cluster reconfigured + // under a live catalog gets a loud error instead (see getOrCreateLakeSibling). + private volatile Connector lakeSibling; + + // The configuration lakeSibling was built from, for the "reconfigured under a live catalog" check. + // Written BEFORE the volatile lakeSibling publishes it, so a reader that sees the sibling sees this. + private Map lakeSiblingProperties; + public FlussConnector(Map properties, ConnectorContext context) { FlussConnectorProperties.validate(properties); this.properties = Collections.unmodifiableMap(new HashMap<>(properties)); this.catalogName = context.getCatalogName(); + this.context = context; } @Override public ConnectorMetadata getMetadata(ConnectorSession session) { return new FlussConnectorMetadata(adminOps(), - FlussConnectorProperties.typeMappingOptions(properties)); + FlussConnectorProperties.typeMappingOptions(properties), + this::getOrCreateLakeSibling, this::lakeSiblingOwning); } /** @@ -71,6 +100,78 @@ public ConnectorScanPlanProvider getScanPlanProvider() { return new FlussScanPlanProvider(adminOps(), properties); } + /** + * Routes scan planning by handle, so a lake table's scan is planned by the paimon sibling that owns + * its handle and reads as a plain paimon table (native readers, its own statistics and time travel). + * A fluss handle keeps this connector's own provider. + */ + @Override + public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) { + Connector owner = lakeSiblingOwning(handle); + return owner == null ? getScanPlanProvider() : owner.getScanPlanProvider(handle); + } + + /** + * Only fluss's own handles. The engine asks this to route a handle back to the connector that made it, + * and a lake table's handle was made by the paimon sibling, which answers for itself. + */ + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof FlussTableHandle; + } + + /** + * The already-built lake sibling when it owns {@code handle}, else null (i.e. a fluss handle). Asks the + * sibling's {@code ownsHandle} because the sibling tests its OWN in-loader handle type, which this side + * cannot {@code instanceof} across the plugin split. + * + *

Deliberately a peek, never a build: a query that never touches a lake table must not construct a + * paimon catalog (nor fail when the paimon plugin is absent). + */ + private Connector lakeSiblingOwning(ConnectorTableHandle handle) { + Connector sibling = lakeSibling; + return sibling != null && sibling.ownsHandle(handle) ? sibling : null; + } + + /** + * Builds (once) and returns this catalog's paimon sibling. Fails loud when no paimon provider is + * available, e.g. the plugin is not installed; that failure is NOT memoized (the field stays unset), so + * a later-installed plugin recovers on the next access. + * + *

Also fails loud when a second, DIFFERENT lake configuration shows up — see the field comment: a + * second paimon sibling could not be routed apart from the first, so serving both would mean reading + * one warehouse under the other's name. The message asks for a catalog refresh, which rebuilds this + * connector and picks up the new configuration. It deliberately does not print either configuration: + * they carry storage credentials. + * + *

Package-private (not private) so a unit test can drive the sibling wiring without + * {@link #getMetadata} first opening a real fluss connection. + */ + Connector getOrCreateLakeSibling(Map siblingProperties) { + if (lakeSibling == null) { + synchronized (this) { + if (lakeSibling == null) { + Connector sibling = + context.createSiblingConnector(PAIMON_CONNECTOR_TYPE, siblingProperties); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot read the lake table of fluss catalog '" + catalogName + + "': the paimon connector plugin is not available"); + } + lakeSiblingProperties = Collections.unmodifiableMap(new HashMap<>(siblingProperties)); + lakeSibling = sibling; + } + } + } + if (!lakeSiblingProperties.equals(siblingProperties)) { + throw new DorisConnectorException( + "Fluss catalog '" + catalogName + "' is already serving lake tables with a different" + + " paimon configuration than this table's. Its fluss cluster was reconfigured;" + + " refresh the catalog to pick up the new lake configuration"); + } + return lakeSibling; + } + @Override public ConnectorTestResult testConnection(ConnectorSession session) { try { @@ -81,17 +182,39 @@ public ConnectorTestResult testConnection(ConnectorSession session) { } } + /** + * Closes this catalog's fluss connection and its lake sibling. The engine closes only a catalog's + * PRIMARY connector, so the sibling's lifecycle is this connector's to own — an unclosed one leaks its + * paimon catalog (file handles, client pools) for the life of the FE. The sibling is closed first and + * its failure is held back, so it cannot leak the fluss connection. + */ @Override public void close() throws IOException { + IOException siblingFailure = null; + Connector sibling = lakeSibling; + lakeSibling = null; + lakeSiblingProperties = null; + if (sibling != null) { + try { + sibling.close(); + } catch (Exception e) { + siblingFailure = new IOException( + "Failed to close the paimon lake sibling of catalog '" + catalogName + "'", e); + } + } + Connection toClose = connection; connection = null; - if (toClose == null) { - return; + if (toClose != null) { + try { + toClose.close(); + } catch (Exception e) { + throw new IOException( + "Failed to close the fluss connection of catalog '" + catalogName + "'", e); + } } - try { - toClose.close(); - } catch (Exception e) { - throw new IOException("Failed to close the fluss connection of catalog '" + catalogName + "'", e); + if (siblingFailure != null) { + throw siblingFailure; } } diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 82b87b349e7ad4..7f4c6a09a08296 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.api.Connector; import org.apache.doris.connector.api.ConnectorColumn; import org.apache.doris.connector.api.ConnectorMetadata; import org.apache.doris.connector.api.ConnectorPartitionInfo; @@ -24,6 +25,7 @@ import org.apache.doris.connector.api.ConnectorTableSchema; import org.apache.doris.connector.api.ConnectorTableStatistics; import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; import org.apache.doris.connector.api.handle.ConnectorColumnHandle; import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.pushdown.ConnectorExpression; @@ -47,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * Fluss metadata for one statement: a thin mapping from the connector SPI onto {@link FlussAdminOps}. @@ -57,6 +60,24 @@ *

Every method that needs the table's schema goes through {@link #tableInfo}, which memoizes the * fetch for the statement — the handle, the schema, the column handles and (later) split planning * therefore see one coherent version of the table for one round trip. + * + *

It is also the gateway for lake tables: a datalake-enabled table exposes a {@code lake} system + * table, so {@code tbl$lake} resolves to a handle made by the embedded paimon sibling and every later + * per-handle call is forwarded to that sibling's metadata. Those forwards are the guarded methods below; + * each one first asks {@link #siblingOwner} whether the handle is foreign, and only then falls through to + * fluss's own implementation. A guard that is missing would not degrade gracefully — fluss's body casts to + * {@link FlussTableHandle} and would throw {@link ClassCastException} on a paimon handle. Every method + * that performs that cast is guarded; a new one must be too. + * + *

The methods this connector does NOT implement are deliberately left unguarded: they inherit the SPI's + * neutral defaults, so a lake handle reaching them is answered "not supported" rather than crashing. That + * costs the lake table paimon's MVCC pin and time travel, and the reason is structural rather than an + * oversight: fe-core picks a table's MVCC-capable class from the FRONT DOOR connector's capabilities + * ({@code SUPPORTS_MVCC_SNAPSHOT}), which fluss does not declare, so the engine never asks for a pin on a + * fluss catalog's table — forwarding those calls today would be unreachable code. Schema and data stay + * consistent (both read latest). Should fluss ever declare that capability, add forwards for + * {@code beginQuerySnapshot} / {@code resolveTimeTravel} / {@code applySnapshot} and the snapshot overloads + * of {@code getTableSchema} / {@code getTableStatistics} in the same breath. */ public class FlussConnectorMetadata implements ConnectorMetadata { @@ -65,12 +86,41 @@ public class FlussConnectorMetadata implements ConnectorMetadata { /** What {@code getTableSchema} reports as the table's format; surfaces in DESCRIBE / EXPLAIN. */ private static final String TABLE_FORMAT_TYPE = "FLUSS"; + /** + * The system-table name that reads a datalake table's lake side, i.e. {@code tbl$lake}. Matches the + * suffix fluss's own Flink catalog uses ({@code FlinkCatalog.LAKE_TABLE_SPLITTER = "$lake"}); the + * engine supplies the {@code $}. + */ + private static final String LAKE_SYS_TABLE = "lake"; + + /** The only lake format that can be delegated today; fluss also defines iceberg / lance / hudi. */ + private static final String PAIMON_LAKE_FORMAT = "paimon"; + private final FlussAdminOps adminOps; private final FlussTypeMapping.Options typeMappingOptions; + private final Function, Connector> lakeSiblingFactory; + private final Function siblingOwner; - public FlussConnectorMetadata(FlussAdminOps adminOps, FlussTypeMapping.Options typeMappingOptions) { + public FlussConnectorMetadata(FlussAdminOps adminOps, FlussTypeMapping.Options typeMappingOptions, + Function, Connector> lakeSiblingFactory, + Function siblingOwner) { this.adminOps = adminOps; this.typeMappingOptions = typeMappingOptions; + this.lakeSiblingFactory = lakeSiblingFactory; + this.siblingOwner = siblingOwner; + } + + /** + * Obtains the lake sibling's metadata through the per-statement funnel: the first forward in a statement + * builds it and every later one reuses that instance, mirroring what fe-core's own metadata funnel does + * for a plain connector. The key carries the catalog id and the owner role because a fluss catalog runs + * two connectors (itself and the paimon sibling) under ONE catalog id — keying on the id alone would + * collapse them onto one metadata and misroute every call. Only SPI types are touched here, and neither + * the returned metadata nor any handle it produces may be cast (cross-loader {@code CCE}). + */ + private ConnectorMetadata siblingMetadata(ConnectorSession session, Connector sibling) { + String key = "metadata:" + session.getCatalogId() + ":" + LAKE_SYS_TABLE; + return session.getStatementScope().getOrCreateMetadata(key, () -> sibling.getMetadata(session)); } @Override @@ -107,8 +157,99 @@ public Optional getTableHandle( } } + /** + * Reports the {@code lake} system table for a table that has a lake side, so {@code tbl$lake} resolves. + * + *

Gated on {@code table.datalake.enabled} alone, not on the lake FORMAT: a table with an unsupported + * lake format still advertises {@code $lake} so that reading it produces + * {@link #getSysTableHandle}'s precise "this format is not supported" error rather than fe-core's + * generic "no such table". A table with no lake at all advertises nothing — announcing a sub-table that + * can only fail would be worse than not offering it. + */ + @Override + public List listSupportedSysTables(ConnectorSession session, + ConnectorTableHandle baseTableHandle) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return siblingMetadata(session, owner).listSupportedSysTables(session, baseTableHandle); + } + FlussTableHandle flussHandle = (FlussTableHandle) baseTableHandle; + return flussHandle.isDataLakeEnabled() + ? Collections.singletonList(LAKE_SYS_TABLE) + : Collections.emptyList(); + } + + /** + * Resolves {@code tbl$lake} to the paimon sibling's handle for the same {@code db.table} name — which is + * the name fluss's tiering service writes the lake table under. From here on the table IS a paimon + * table: the engine routes its scan by handle to the sibling's plan provider, and this metadata's + * guarded methods forward the rest. + * + *

The sibling is configured from THIS table's properties, where the fluss coordinator puts the + * cluster's lake settings; see {@link PaimonSiblingProperties}. + */ + @Override + public Optional getSysTableHandle(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return siblingMetadata(session, owner).getSysTableHandle(session, baseTableHandle, sysName); + } + if (!LAKE_SYS_TABLE.equals(sysName)) { + return Optional.empty(); + } + + FlussTableHandle flussHandle = (FlussTableHandle) baseTableHandle; + // Re-checked rather than assumed from listSupportedSysTables: discovery and resolution are two + // round trips, and the table could have had its lake turned off in between. + if (!flussHandle.isDataLakeEnabled()) { + throw new DorisConnectorException("Table '" + flussHandle.getDatabaseName() + "." + + flussHandle.getTableName() + "' has no lake table: it is not created with" + + " table.datalake.enabled = true"); + } + String lakeFormat = flussHandle.getDataLakeFormat(); + if (lakeFormat == null || !PAIMON_LAKE_FORMAT.equalsIgnoreCase(lakeFormat)) { + throw new DorisConnectorException("Cannot read the lake table of '" + + flussHandle.getDatabaseName() + "." + flussHandle.getTableName() + + "': its table.datalake.format is '" + lakeFormat + + "', and the fluss connector currently supports only '" + PAIMON_LAKE_FORMAT + "'"); + } + + Connector sibling = lakeSiblingFactory.apply( + PaimonSiblingProperties.synthesize(flussHandle.getProperties())); + Optional lakeHandle = siblingMetadata(session, sibling).getTableHandle( + session, flussHandle.getDatabaseName(), flussHandle.getTableName()); + if (!lakeHandle.isPresent()) { + // The lake table is created by the tiering service on its first commit, so "not there" means + // nothing has been tiered yet — a state that resolves itself and is worth saying out loud. + // Returning empty here would instead surface as fe-core's generic "no such table", pointing the + // user at a name that IS correct. + throw new DorisConnectorException("The lake table of '" + flussHandle.getDatabaseName() + "." + + flussHandle.getTableName() + "' does not exist yet: nothing has been tiered to the" + + " lake. Start (or wait for) the fluss tiering service for this table"); + } + return lakeHandle; + } + + @Override + public boolean isPartitionValuesSysTable(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return siblingMetadata(session, owner) + .isPartitionValuesSysTable(session, baseTableHandle, sysName); + } + // The lake table is a real data table served by the paimon sibling, not the generic + // partition_values function. + return false; + } + @Override public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return siblingMetadata(session, owner).getTableSchema(session, handle); + } FlussTableHandle flussHandle = (FlussTableHandle) handle; TableInfo info = tableInfo(session, flussHandle.toTablePath()); @@ -135,6 +276,10 @@ public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTa @Override public Map getColumnHandles( ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return siblingMetadata(session, owner).getColumnHandles(session, handle); + } FlussTableHandle flussHandle = (FlussTableHandle) handle; List columns = tableInfo(session, flussHandle.toTablePath()).getSchema().getColumns(); Map handles = new LinkedHashMap<>(); @@ -152,6 +297,10 @@ public String getTableComment(ConnectorSession session, String dbName, String ta @Override public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return siblingMetadata(session, owner).listPartitionNames(session, handle); + } List partitions = listPartitions(session, handle, Optional.empty()); List names = new ArrayList<>(partitions.size()); for (ConnectorPartitionInfo partition : partitions) { @@ -175,6 +324,10 @@ public List listPartitionNames(ConnectorSession session, ConnectorTableH @Override public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, Optional filter) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return siblingMetadata(session, owner).listPartitions(session, handle, filter); + } FlussTableHandle flussHandle = (FlussTableHandle) handle; List partitionKeys = flussHandle.getPartitionKeys(); if (partitionKeys.isEmpty()) { @@ -208,6 +361,10 @@ public List listPartitions(ConnectorSession session, @Override public Optional getTableStatistics( ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return siblingMetadata(session, owner).getTableStatistics(session, handle); + } FlussTableHandle flussHandle = (FlussTableHandle) handle; long rowCount; try { @@ -228,6 +385,11 @@ public Optional getTableStatistics( * scan node the lake connectors use (the fluss ranges ride in its format-specific descriptor), so * the table descriptor is the generic hive-shaped one those connectors send, exactly as paimon and * hudi do; a fluss-specific Thrift table type would buy nothing the scan path reads. + * + *

Not guarded for a lake handle, and it could not be: the signature carries names, not a handle. It + * needs no guard because the paimon connector builds the byte-identical descriptor + * ({@code PaimonConnectorMetadata.buildTableDescriptor}) — verified, not assumed. If either side's + * descriptor ever diverges, this becomes a real gap that a handle-less signature cannot close here. */ @Override public TTableDescriptor buildTableDescriptor(ConnectorSession session, diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java new file mode 100644 index 00000000000000..4781f131c157ff --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Synthesizes the catalog-property map for the embedded paimon sibling connector that serves a + * fluss lake table ({@code tbl$lake}). Mirrors the hive gateway's {@code HudiSiblingProperties} / + * {@code IcebergSiblingProperties}: the sibling is built through + * {@code ConnectorContext.createSiblingConnector("paimon", synthesize(...))} so its classes come from the + * paimon plugin's own loader — the fluss plugin bundles no paimon at all. + * + *

The one structural difference from hive: hive's lake configuration is part of the CATALOG properties + * the user typed, so its synthesis is a verbatim copy. Fluss's lives in the TABLE properties instead — the + * fluss coordinator merges the cluster-level {@code datalake.paimon.*} settings into every datalake table's + * properties under a {@code table.} prefix, and only while {@code table.datalake.enabled} is true + * ({@code TableRegistration#toTableInfo}). So the input here is a table's property map and the mapping has + * to strip that prefix: + * + * 

+ *   fluss cluster config      datalake.paimon.warehouse = /lake
+ *   fluss table property      table.datalake.paimon.warehouse = /lake
+ *   paimon catalog property   warehouse = /lake
+ *
+ * + *

Two keys are renamed because Doris's paimon connector spells them differently from paimon itself; + * everything else is forwarded with only the prefix removed. That verbatim tail is what carries a real + * deployment's storage keys ({@code fs.*} / {@code dfs.*} / {@code hadoop.*}), which the paimon connector + * reads under exactly those names. + */ +final class PaimonSiblingProperties { + + /** + * The prefix the fluss coordinator gives its injected paimon lake settings inside a table's property + * map. + */ + private static final String LAKE_OPTION_PREFIX = "table.datalake.paimon."; + + /** Paimon's own name for the metastore flavor; Doris's paimon connector calls it paimon.catalog.type. */ + private static final String FLUSS_METASTORE = "metastore"; + private static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; + + /** Same name on both sides, but required, so it is handled explicitly rather than by the tail. */ + private static final String WAREHOUSE = "warehouse"; + + /** The only metastore flavor this connector serves today (see the class comment of the gate). */ + private static final String FILESYSTEM = "filesystem"; + + private PaimonSiblingProperties() { + } + + /** + * Returns a NEW paimon catalog-property map derived from one fluss table's properties. The input is + * never mutated and is not aliased into the result. + * + *

Fails loud on a lake configuration this connector cannot serve, rather than handing the paimon + * connector a half-built map and letting it fail with a message about a catalog nobody created. + */ + static Map synthesize(Map flussTableProperties) { + Map lakeOptions = new LinkedHashMap<>(); + for (Map.Entry entry : flussTableProperties.entrySet()) { + if (entry.getKey().startsWith(LAKE_OPTION_PREFIX)) { + lakeOptions.put(entry.getKey().substring(LAKE_OPTION_PREFIX.length()), entry.getValue()); + } + } + + String metastore = lakeOptions.remove(FLUSS_METASTORE); + // Absent means paimon's own default, which is filesystem — the same thing this connector supports. + if (metastore != null && !FILESYSTEM.equalsIgnoreCase(metastore)) { + throw new DorisConnectorException( + "Cannot read the lake table: its fluss cluster configures a '" + metastore + + "' paimon catalog (datalake.paimon." + FLUSS_METASTORE + + "), and the fluss connector currently supports only '" + FILESYSTEM + "'"); + } + + String warehouse = lakeOptions.remove(WAREHOUSE); + if (warehouse == null || warehouse.isEmpty()) { + throw new DorisConnectorException( + "Cannot read the lake table: the fluss table carries no '" + + LAKE_OPTION_PREFIX + WAREHOUSE + + "'. The fluss cluster must configure datalake.paimon." + WAREHOUSE + + " for its lake tables to be readable"); + } + + // LinkedHashMap so a failure message or a log line renders the same order every time. + Map siblingProperties = new LinkedHashMap<>(); + siblingProperties.put(PAIMON_CATALOG_TYPE, FILESYSTEM); + siblingProperties.put(WAREHOUSE, warehouse); + siblingProperties.putAll(lakeOptions); + return siblingProperties; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java new file mode 100644 index 00000000000000..b3f498fd9ee91c --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The connector's ownership of its lake sibling: how it is built, cached, routed to and closed. + * + *

None of this touches a fluss cluster, which is also what the production code guarantees: a query that + * never reads a lake table must not build a paimon catalog, and reaching a lake table's scan provider must + * not open a fluss connection. + */ +public class FlussConnectorLakeSiblingTest { + + /** A lake configuration, i.e. what {@code PaimonSiblingProperties} hands the factory. */ + private static Map lakeProperties(String warehouse) { + Map properties = new HashMap<>(); + properties.put("paimon.catalog.type", "filesystem"); + properties.put("warehouse", warehouse); + return properties; + } + + /** Records what the engine was asked to build, and answers with whatever the test staged. */ + private static final class RecordingContext implements ConnectorContext { + + private final List requestedTypes = new ArrayList<>(); + private final List> requestedProperties = new ArrayList<>(); + private boolean providerAvailable = true; + + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + + @Override + public Connector createSiblingConnector(String catalogType, Map properties) { + requestedTypes.add(catalogType); + requestedProperties.add(properties); + return providerAvailable ? new RecordingLakeSibling(properties) : null; + } + } + + private static FlussConnector connector(ConnectorContext context) { + Map catalogProperties = new HashMap<>(); + catalogProperties.put("fluss.bootstrap.servers", "127.0.0.1:9123"); + return new FlussConnector(catalogProperties, context); + } + + @Test + public void siblingIsBuiltAsAPaimonConnectorFromTheGivenProperties() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + Map properties = lakeProperties("/lake"); + Connector sibling = connector.getOrCreateLakeSibling(properties); + + Assertions.assertNotNull(sibling); + // The type string is what the engine matches against the installed plugins; the fluss plugin + // bundles no paimon class at all, so this literal is the whole contract. + Assertions.assertEquals(Collections.singletonList("paimon"), context.requestedTypes); + Assertions.assertEquals(properties, context.requestedProperties.get(0)); + } + + @Test + public void oneLakeConfigurationBuildsOneSibling() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + Connector first = connector.getOrCreateLakeSibling(lakeProperties("/lake")); + Connector second = connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Equal-by-value properties, not the same map instance: every statement synthesizes a fresh map + // from the table's properties, so an identity-keyed cache would build a paimon catalog per query. + Assertions.assertSame(first, second); + Assertions.assertEquals(1, context.requestedTypes.size()); + } + + @Test + public void secondLakeConfigurationFailsLoud() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Two paimon siblings answer "is this handle yours?" with the same class test, so the second could + // never be routed apart from the first — a table would silently read the wrong warehouse. Refusing + // is the only honest answer; the message points at the fix (refresh the catalog). + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getOrCreateLakeSibling(lakeProperties("/other"))); + Assertions.assertTrue(failure.getMessage().contains("refresh the catalog"), failure.getMessage()); + // Storage credentials live in these maps; a message that dumps them ends up in the FE audit log. + Assertions.assertFalse(failure.getMessage().contains("/other"), failure.getMessage()); + Assertions.assertEquals(1, context.requestedTypes.size(), "the refused one must not be built"); + } + + @Test + public void missingPaimonPluginFailsLoudAndIsNotCached() { + RecordingContext context = new RecordingContext(); + context.providerAvailable = false; + FlussConnector connector = connector(context); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getOrCreateLakeSibling(lakeProperties("/lake"))); + Assertions.assertTrue(failure.getMessage().contains("paimon"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("fluss_catalog"), failure.getMessage()); + + // Not memoizing the failure is what lets a later-installed plugin recover without a catalog + // refresh; a cached null would keep answering "not available" forever. + context.providerAvailable = true; + Assertions.assertNotNull(connector.getOrCreateLakeSibling(lakeProperties("/lake"))); + } + + @Test + public void lakeHandleIsPlannedByTheSibling() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + RecordingLakeSibling sibling = + (RecordingLakeSibling) connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Routing is by handle OWNERSHIP, asked of the sibling — the fluss side cannot recognize a foreign + // handle by class across the plugin split, so "is it yours?" is the only question available. + Assertions.assertSame(sibling.scanPlanProvider, + connector.getScanPlanProvider(new RecordingLakeSibling.Handle("db", "t"))); + } + + @Test + public void onlyFlussHandlesAreOwnedByThisConnector() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + // The engine asks this to route a handle back to the connector that made it. Claiming a sibling's + // handle would make the engine plan a paimon table with fluss's planner. + Assertions.assertFalse(connector.ownsHandle(new RecordingLakeSibling.Handle("db", "t"))); + } + + @Test + public void closingTheConnectorClosesTheSibling() throws IOException { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + RecordingLakeSibling sibling = + (RecordingLakeSibling) connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + connector.close(); + + Assertions.assertTrue(sibling.closed); + // And the reference must be dropped, or a reopened catalog would keep routing to a closed sibling. + // A now-different lake configuration being accepted is what proves the slot is free again. + Assertions.assertNotSame(sibling, connector.getOrCreateLakeSibling(lakeProperties("/other"))); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java index a74c536695c0e6..76c949b81bf36f 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java @@ -60,7 +60,21 @@ public class FlussConnectorMetadataTest { private static final TablePath PK_TABLE = TablePath.of("db", "pk_table"); private static FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { - return new FlussConnectorMetadata(adminOps, FlussTypeMapping.Options.DEFAULT); + return metadata(adminOps, FlussTypeMapping.Options.DEFAULT); + } + + /** + * A metadata with the lake seams wired to "there is no lake": the sibling owner never claims a handle + * and the factory refuses to build one. A test in this class that unexpectedly took the lake path + * therefore fails loudly instead of quietly exercising a half-built sibling. + */ + private static FlussConnectorMetadata metadata( + RecordingFlussAdminOps adminOps, FlussTypeMapping.Options options) { + return new FlussConnectorMetadata(adminOps, options, + properties -> { + throw new AssertionError("no lake sibling is expected in this test"); + }, + handle -> null); } /** A non-partitioned log table: two columns, one of them commented, three buckets. */ @@ -242,7 +256,7 @@ public void timestampWithLocalTimeZoneKeepsItsMarkerUnderEitherMapping() { .build()); for (boolean mapTimestampTz : new boolean[] {false, true}) { - FlussConnectorMetadata metadata = new FlussConnectorMetadata( + FlussConnectorMetadata metadata = metadata( adminOps, new FlussTypeMapping.Options(false, mapTimestampTz)); ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") .orElseThrow(AssertionError::new); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java new file mode 100644 index 00000000000000..74fda23b49a79f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java @@ -0,0 +1,319 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The lake gateway: {@code tbl$lake} is served by an embedded paimon sibling, and every per-handle call + * about that table has to reach the sibling instead of fluss. + * + *

The sibling is a hand-written {@link RecordingLakeSibling} (this module bans mock frameworks) whose + * answers are values fluss could never produce, so an assertion that finds them proves the forward really + * happened rather than that both sides agree by coincidence. + */ +public class FlussLakeTableTest { + + private static final TablePath LAKE_TABLE = TablePath.of("db", "lake_table"); + private static final TablePath PLAIN_TABLE = TablePath.of("db", "plain_table"); + + /** The sibling connectors this test's factory built, in build order. */ + private final List builtSiblings = new ArrayList<>(); + + private final ConnectorSession session = new FlussTestSession(7L, "q1"); + + /** + * A fluss table with a paimon lake, exactly as a coordinator reports one: the lake switch, the format, + * and the cluster's paimon settings injected under {@code table.datalake.paimon.}. + */ + private RecordingFlussAdminOps withLakeTable() { + return withLakeTable("paimon"); + } + + private RecordingFlussAdminOps withLakeTable(String lakeFormat) { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LAKE_TABLE, FlussTestTables.builder(LAKE_TABLE) + .column("id", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .buckets(2) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", lakeFormat) + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + adminOps.tableInfos.put(PLAIN_TABLE, FlussTestTables.builder(PLAIN_TABLE) + .column("id", DataTypes.BIGINT()) + .buckets(1) + .build()); + return adminOps; + } + + /** + * Metadata whose lake seams are the real ones a connector supplies: a factory that builds (and + * records) a sibling, and an owner resolver that answers exactly the way {@code FlussConnector}'s does + * — by asking each built sibling whether the handle is its own. + */ + private FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { + return new FlussConnectorMetadata(adminOps, FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + builtSiblings.add(sibling); + return sibling; + }, + handle -> { + for (RecordingLakeSibling sibling : builtSiblings) { + if (sibling.ownsHandle(handle)) { + return sibling; + } + } + return null; + }); + } + + private ConnectorTableHandle baseHandle(FlussConnectorMetadata metadata, TablePath tablePath) { + return metadata.getTableHandle(session, tablePath.getDatabaseName(), tablePath.getTableName()) + .orElseThrow(AssertionError::new); + } + + private ConnectorTableHandle lakeHandle(FlussConnectorMetadata metadata) { + return metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake") + .orElseThrow(AssertionError::new); + } + + @Test + public void datalakeTableOffersTheLakeSystemTable() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // This listing is what makes fe-core resolve the name "lake_table$lake" at all. + Assertions.assertEquals(Collections.singletonList("lake"), + metadata.listSupportedSysTables(session, baseHandle(metadata, LAKE_TABLE))); + } + + @Test + public void tableWithoutALakeOffersNoSystemTable() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Advertising $lake on a table that has none would offer a sub-table whose only possible outcome + // is an error; fe-core's "no such table" is the honest answer. + Assertions.assertTrue( + metadata.listSupportedSysTables(session, baseHandle(metadata, PLAIN_TABLE)).isEmpty()); + } + + @Test + public void theLakeHandleIsTheSiblingsHandleForTheSameName() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // The lake table lives under the SAME db.table name in paimon — that is where fluss's tiering + // service writes it. A handle of the sibling's own type is also what routes every later call. + Assertions.assertTrue(handle instanceof RecordingLakeSibling.Handle); + Assertions.assertEquals("db", ((RecordingLakeSibling.Handle) handle).dbName); + Assertions.assertEquals("lake_table", ((RecordingLakeSibling.Handle) handle).tableName); + Assertions.assertEquals(1, builtSiblings.size(), "one lake configuration, one sibling"); + Assertions.assertEquals(Collections.singletonList("getTableHandle:db.lake_table"), + builtSiblings.get(0).calls); + } + + @Test + public void theSiblingIsConfiguredFromTheTablesLakeProperties() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + lakeHandle(metadata); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + // Whole-map equality: the sibling is a real catalog and a leaked fluss property would either be + // rejected by it or silently reinterpreted as a paimon option. + Assertions.assertEquals(expected, builtSiblings.get(0).properties); + } + + @Test + public void anUnknownSystemTableNameIsNotServed() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Only "lake" is this connector's; asking for anything else must not build a sibling. + Assertions.assertEquals(Optional.empty(), + metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "snapshots")); + Assertions.assertTrue(builtSiblings.isEmpty()); + } + + @Test + public void tableWithoutALakeFailsLoudOnTheLakeHandle() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Reachable when the table's lake is turned off between the listing and the resolution: two round + // trips, so the gate is re-checked rather than assumed. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, PLAIN_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("table.datalake.enabled"), + failure.getMessage()); + } + + @Test + public void anUnsupportedLakeFormatFailsLoud() { + FlussConnectorMetadata metadata = metadata(withLakeTable("iceberg")); + // The reason $lake is offered for ANY lake format: this precise message beats fe-core's generic + // "no such table" for a user whose table is tiered to a lake this connector cannot read yet. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("iceberg"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("paimon"), failure.getMessage()); + Assertions.assertTrue(builtSiblings.isEmpty(), "an unreadable format must not build a catalog"); + } + + @Test + public void anUntieredLakeTableFailsLoud() { + FlussConnectorMetadata metadata = new FlussConnectorMetadata(withLakeTable(), + FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + sibling.lakeTableExists = false; + builtSiblings.add(sibling); + return sibling; + }, + handle -> null); + + // The lake table is only created on the tiering service's first commit. Answering "no such table" + // would send the user after a name that is in fact correct; naming tiering is the actionable form. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("tiering"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("db.lake_table"), failure.getMessage()); + } + + @Test + public void theSchemaOfALakeTableComesFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableSchema schema = metadata.getTableSchema(session, lakeHandle(metadata)); + + // Fluss's own schema for this table is (id, name) with format FLUSS; every value here is the + // sibling's, so the forward is what produced them. + Assertions.assertEquals(RecordingLakeSibling.SCHEMA_TABLE_NAME, schema.getTableName()); + Assertions.assertEquals(RecordingLakeSibling.FORMAT_TYPE, schema.getTableFormatType()); + Assertions.assertEquals(1, schema.getColumns().size()); + Assertions.assertEquals(RecordingLakeSibling.COLUMN_NAME, schema.getColumns().get(0).getName()); + } + + @Test + public void theColumnHandlesOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + Assertions.assertEquals( + Collections.singleton(RecordingLakeSibling.COLUMN_NAME), + metadata.getColumnHandles(session, lakeHandle(metadata)).keySet()); + } + + @Test + public void thePartitionsOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // Fluss's own answer for this (unpartitioned) table is an empty list, so a missing guard would + // silently report "no partitions" for a partitioned lake table and read the whole thing. + List partitions = + metadata.listPartitions(session, handle, Optional.empty()); + Assertions.assertEquals(1, partitions.size()); + Assertions.assertEquals(RecordingLakeSibling.PARTITION_NAME, partitions.get(0).getPartitionName()); + + Assertions.assertEquals( + Collections.singletonList(RecordingLakeSibling.PARTITION_NAME), + metadata.listPartitionNames(session, handle)); + // listPartitionNames must reach the sibling's own method, not derive names from listPartitions: + // the sibling may answer the two differently (and more cheaply). + Assertions.assertTrue(builtSiblings.get(0).calls.contains("listPartitionNames")); + } + + @Test + public void theStatisticsOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Fluss's own path would call getTableStats on the fluss table (an unprogrammed recording call, + // which throws), so this asserts the forward AND that fluss's remote call never happened. + Assertions.assertEquals(RecordingLakeSibling.ROW_COUNT, + metadata.getTableStatistics(session, lakeHandle(metadata)) + .orElseThrow(AssertionError::new).getRowCount()); + } + + @Test + public void systemTableQuestionsAboutALakeTableGoToTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // A lake handle reaching fluss's own bodies would ClassCastException, so these guards are about + // not crashing as much as about delegating: paimon owns its table's system tables. + Assertions.assertEquals(Collections.singletonList("snapshots"), + metadata.listSupportedSysTables(session, handle)); + Assertions.assertEquals(Optional.empty(), + metadata.getSysTableHandle(session, handle, "snapshots")); + Assertions.assertTrue(metadata.isPartitionValuesSysTable(session, handle, "partitions")); + } + + @Test + public void flussHandleStillGetsFlussAnswers() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Build the sibling first, so the guards have a sibling to (wrongly) route to if they matched on + // anything other than handle ownership. + lakeHandle(metadata); + + ConnectorTableSchema schema = metadata.getTableSchema(session, baseHandle(metadata, LAKE_TABLE)); + Assertions.assertEquals("lake_table", schema.getTableName()); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + Assertions.assertEquals(2, schema.getColumns().size()); + } + + @Test + public void oneStatementBuildsOneSiblingMetadata() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + metadata.getTableSchema(session, handle); + metadata.getColumnHandles(session, handle); + metadata.listPartitions(session, handle, Optional.empty()); + metadata.getTableStatistics(session, handle); + + // The per-statement funnel: the sibling's metadata is built once and every forward reuses it, + // exactly as fe-core's own funnel does for a plain connector. Rebuilding per call would re-open + // whatever that metadata memoizes (a schema, a snapshot) and let one statement see two versions. + Assertions.assertEquals(1, builtSiblings.get(0).metadataBuilds); + } + + @Test + public void twoStatementsDoNotShareTheSiblingMetadata() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + metadata.getTableSchema(session, handle); + + // A different statement scope must not reuse the previous statement's metadata: the funnel is + // keyed inside the scope, so this proves the key is not accidentally global. + ConnectorSession other = new FlussTestSession(7L, "q2"); + metadata.getTableSchema(other, handle); + + Assertions.assertEquals(2, builtSiblings.get(0).metadataBuilds); + } + +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java index f40bcf1715fbdf..386ee8a9f2e53f 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -516,7 +516,11 @@ private Map catalog(String key, String value) { } private FlussConnectorMetadata metadata() { - return new FlussConnectorMetadata(adminOps, new FlussTypeMapping.Options(false, false)); + return new FlussConnectorMetadata(adminOps, new FlussTypeMapping.Options(false, false), + properties -> { + throw new AssertionError("no lake sibling is expected in this test"); + }, + handle -> null); } private ConnectorTableHandle handle(TablePath tablePath) { diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java new file mode 100644 index 00000000000000..5f4bbdcabd6121 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The translation from a fluss table's properties to the paimon sibling's catalog properties. + * + *

Assertions compare the WHOLE map, not a key or two: the sibling is handed exactly this map and + * nothing else, so a leaked fluss property or a dropped storage key is a bug either way, and only whole-map + * equality catches both. The keys are written out as literals rather than referenced from the class under + * test — a renamed constant must break these tests, since the names are what the paimon connector reads. + */ +public class PaimonSiblingPropertiesTest { + + /** The properties a fluss coordinator injects for a filesystem-backed paimon lake. */ + private static Map flussTableProperties() { + Map properties = new LinkedHashMap<>(); + properties.put("table.datalake.enabled", "true"); + properties.put("table.datalake.format", "paimon"); + properties.put("table.datalake.paimon.metastore", "filesystem"); + properties.put("table.datalake.paimon.warehouse", "/lake/warehouse"); + return properties; + } + + @Test + public void lakeOptionsBecomeThePaimonCatalogProperties() { + Map expected = new HashMap<>(); + // Paimon calls the flavor "metastore"; Doris's paimon connector calls it paimon.catalog.type. + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, + PaimonSiblingProperties.synthesize(flussTableProperties())); + } + + @Test + public void nonLakePropertiesAreNotForwarded() { + Map properties = flussTableProperties(); + // The kind of thing a fluss table carries alongside its lake settings. Handing these to the paimon + // connector would make it reject the catalog (or, worse, silently treat them as paimon options). + properties.put("bucket.num", "8"); + properties.put("table.datalake.freshness", "3min"); + properties.put("table.log.ttl", "7d"); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties), + "only the table.datalake.paimon.* namespace describes the lake catalog"); + } + + @Test + public void remainingLakeOptionsKeepTheirNamesWithoutThePrefix() { + Map properties = flussTableProperties(); + // A real object-store deployment: the paimon connector reads these storage keys under exactly + // these names, so the prefix has to come off and nothing may be added. + properties.put("table.datalake.paimon.fs.s3a.endpoint", "http://minio:9000"); + properties.put("table.datalake.paimon.fs.s3a.access.key", "ak"); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + expected.put("fs.s3a.endpoint", "http://minio:9000"); + expected.put("fs.s3a.access.key", "ak"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void anAbsentMetastoreIsTheFilesystemDefault() { + Map properties = flussTableProperties(); + properties.remove("table.datalake.paimon.metastore"); + + // Paimon's own default is filesystem, so silence means the flavor this connector supports. The + // catalog type must still be stated explicitly: the paimon connector defaults on its own, and + // leaving it unset would make the sibling depend on that default staying put. + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void anUnsupportedMetastoreFailsLoud() { + Map properties = flussTableProperties(); + properties.put("table.datalake.paimon.metastore", "hive"); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + // Naming both the flavor found and the one supported is what makes this actionable: the fix is in + // the fluss cluster's configuration, not in Doris. + Assertions.assertTrue(failure.getMessage().contains("hive"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("filesystem"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("datalake.paimon.metastore"), + failure.getMessage()); + } + + @Test + public void missingWarehouseFailsLoud() { + Map properties = flussTableProperties(); + properties.remove("table.datalake.paimon.warehouse"); + + // Without a warehouse the paimon catalog cannot be built at all. Failing here names the fluss + // setting to fix; letting it through would surface as a paimon error about a catalog nobody created. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + Assertions.assertTrue(failure.getMessage().contains("table.datalake.paimon.warehouse"), + failure.getMessage()); + } + + @Test + public void anEmptyWarehouseIsTreatedAsMissing() { + Map properties = flussTableProperties(); + properties.put("table.datalake.paimon.warehouse", ""); + + Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void theInputMapIsNeverMutated() { + Map properties = flussTableProperties(); + Map before = new LinkedHashMap<>(properties); + + PaimonSiblingProperties.synthesize(properties); + + // The caller's map is a table handle's property map, shared and unmodifiable in production; a + // synthesis that consumed entries from it would corrupt every later read of that table. + Assertions.assertEquals(before, properties); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java new file mode 100644 index 00000000000000..a15d8a3013c5df --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A stand-in for the embedded paimon sibling connector, hand written because this module bans mock + * frameworks. It plays the part that matters for the gateway: it owns a handle type of its OWN + * ({@link Handle}) that the fluss code can never recognize by class, and it records every call it is + * forwarded so a test can assert the forward actually happened — and reached the sibling, not fluss. + * + *

Every answer is a distinctive constant, so an assertion that finds it can only have come through the + * sibling; a test that passed because fluss answered instead would see fluss's own values. + */ +final class RecordingLakeSibling implements Connector { + + /** The sibling's own handle type: the analogue of {@code PaimonTableHandle}. */ + static final class Handle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + + final String dbName; + final String tableName; + + Handle(String dbName, String tableName) { + this.dbName = dbName; + this.tableName = tableName; + } + } + + /** The table name reported in the schema; a value fluss's own path could never produce. */ + static final String SCHEMA_TABLE_NAME = "lake_side_table"; + static final String FORMAT_TYPE = "PAIMON"; + static final String COLUMN_NAME = "lake_only_column"; + static final String PARTITION_NAME = "dt=lake"; + static final long ROW_COUNT = 4242L; + + /** The scan plan provider this sibling hands out; identity is what the routing test asserts. */ + final ConnectorScanPlanProvider scanPlanProvider = (session, request) -> Collections.emptyList(); + + final Map properties; + final List calls = new ArrayList<>(); + + /** When false, the sibling reports the lake table as absent (nothing tiered to it yet). */ + boolean lakeTableExists = true; + + /** How many metadata instances this sibling was asked to build (the per-statement funnel's proof). */ + int metadataBuilds; + boolean closed; + + RecordingLakeSibling(Map properties) { + this.properties = properties; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + metadataBuilds++; + return new Metadata(); + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return scanPlanProvider; + } + + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof Handle; + } + + @Override + public void close() { + closed = true; + } + + /** The sibling's metadata: records the forward, then answers with its own distinctive values. */ + private final class Metadata implements ConnectorMetadata { + + @Override + public Optional getTableHandle( + ConnectorSession session, String dbName, String tableName) { + calls.add("getTableHandle:" + dbName + "." + tableName); + return lakeTableExists ? Optional.of(new Handle(dbName, tableName)) : Optional.empty(); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getTableSchema"); + List columns = Collections.singletonList( + new ConnectorColumn(COLUMN_NAME, ConnectorType.of("INT"), "", true, null, true)); + return new ConnectorTableSchema( + SCHEMA_TABLE_NAME, columns, FORMAT_TYPE, Collections.emptyMap()); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getColumnHandles"); + Map handles = new LinkedHashMap<>(); + handles.put(COLUMN_NAME, new FlussColumnHandle(COLUMN_NAME, 0)); + return handles; + } + + @Override + public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + calls.add("listPartitionNames"); + return Collections.singletonList(PARTITION_NAME); + } + + @Override + public List listPartitions(ConnectorSession session, + ConnectorTableHandle handle, Optional filter) { + calls.add("listPartitions"); + return Collections.singletonList(new ConnectorPartitionInfo( + PARTITION_NAME, Collections.singletonMap("dt", "lake"), Collections.emptyMap(), + Collections.singletonList("lake"), Collections.emptyList())); + } + + @Override + public Optional getTableStatistics( + ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getTableStatistics"); + return Optional.of(new ConnectorTableStatistics(ROW_COUNT, -1)); + } + + @Override + public List listSupportedSysTables(ConnectorSession session, + ConnectorTableHandle baseTableHandle) { + calls.add("listSupportedSysTables"); + return Collections.singletonList("snapshots"); + } + + @Override + public Optional getSysTableHandle(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + calls.add("getSysTableHandle:" + sysName); + return Optional.empty(); + } + + @Override + public boolean isPartitionValuesSysTable(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + calls.add("isPartitionValuesSysTable:" + sysName); + return true; + } + } +} From a3ae2b70ab15c56c3ee610b8143b195b723813ad Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 10:56:57 +0800 Subject: [PATCH 17/35] [fix](fluss) Call the lake sibling under its own classloader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine pins the context classloader to the plugin whose object it is about to call, so everything the fluss connector runs sees the fluss plugin's loader. That is right until the connector calls its lake sibling: the sibling's SDK is loaded child-first by ITS plugin, and the parts of it that discover implementations by ServiceLoader — catalog factories, file IO, file formats — look them up through the context classloader, where none of them exist. The failure would surface as a NoClassDefFoundError at the first lake table read rather than at wiring time. Route every forward through one helper that pins the loader and restores the caller's, including on a throw. Building the sibling's metadata is inside the pin too: that call already opens the lake catalog. The helper also owns the per-statement metadata memo, so the pin and the shared instance cannot be forgotten by a new caller. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussConnectorMetadata.java | 37 ++--- .../doris/connector/fluss/LakeSibling.java | 83 ++++++++++ .../connector/fluss/LakeSiblingTest.java | 152 ++++++++++++++++++ 3 files changed, 252 insertions(+), 20 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 7f4c6a09a08296..8881673b43a19b 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -111,16 +111,13 @@ public FlussConnectorMetadata(FlussAdminOps adminOps, FlussTypeMapping.Options t } /** - * Obtains the lake sibling's metadata through the per-statement funnel: the first forward in a statement - * builds it and every later one reuses that instance, mirroring what fe-core's own metadata funnel does - * for a plain connector. The key carries the catalog id and the owner role because a fluss catalog runs - * two connectors (itself and the paimon sibling) under ONE catalog id — keying on the id alone would - * collapse them onto one metadata and misroute every call. Only SPI types are touched here, and neither - * the returned metadata nor any handle it produces may be cast (cross-loader {@code CCE}). + * Forwards one call to the lake sibling's metadata. Only SPI types are touched, and neither the metadata + * nor any handle it produces may be cast (cross-loader {@code CCE}). {@link LakeSibling#forward} owns the + * classloader pin and the per-statement instance, so both are shared with the scan planner. */ - private ConnectorMetadata siblingMetadata(ConnectorSession session, Connector sibling) { - String key = "metadata:" + session.getCatalogId() + ":" + LAKE_SYS_TABLE; - return session.getStatementScope().getOrCreateMetadata(key, () -> sibling.getMetadata(session)); + private T forward(ConnectorSession session, Connector sibling, + Function call) { + return LakeSibling.forward(session, sibling, call); } @Override @@ -171,7 +168,7 @@ public List listSupportedSysTables(ConnectorSession session, ConnectorTableHandle baseTableHandle) { Connector owner = siblingOwner.apply(baseTableHandle); if (owner != null) { - return siblingMetadata(session, owner).listSupportedSysTables(session, baseTableHandle); + return forward(session, owner, m -> m.listSupportedSysTables(session, baseTableHandle)); } FlussTableHandle flussHandle = (FlussTableHandle) baseTableHandle; return flussHandle.isDataLakeEnabled() @@ -193,7 +190,7 @@ public Optional getSysTableHandle(ConnectorSession session ConnectorTableHandle baseTableHandle, String sysName) { Connector owner = siblingOwner.apply(baseTableHandle); if (owner != null) { - return siblingMetadata(session, owner).getSysTableHandle(session, baseTableHandle, sysName); + return forward(session, owner, m -> m.getSysTableHandle(session, baseTableHandle, sysName)); } if (!LAKE_SYS_TABLE.equals(sysName)) { return Optional.empty(); @@ -217,8 +214,8 @@ public Optional getSysTableHandle(ConnectorSession session Connector sibling = lakeSiblingFactory.apply( PaimonSiblingProperties.synthesize(flussHandle.getProperties())); - Optional lakeHandle = siblingMetadata(session, sibling).getTableHandle( - session, flussHandle.getDatabaseName(), flussHandle.getTableName()); + Optional lakeHandle = forward(session, sibling, m -> m.getTableHandle( + session, flussHandle.getDatabaseName(), flussHandle.getTableName())); if (!lakeHandle.isPresent()) { // The lake table is created by the tiering service on its first commit, so "not there" means // nothing has been tiered yet — a state that resolves itself and is worth saying out loud. @@ -236,8 +233,8 @@ public boolean isPartitionValuesSysTable(ConnectorSession session, ConnectorTableHandle baseTableHandle, String sysName) { Connector owner = siblingOwner.apply(baseTableHandle); if (owner != null) { - return siblingMetadata(session, owner) - .isPartitionValuesSysTable(session, baseTableHandle, sysName); + return forward(session, owner, + m -> m.isPartitionValuesSysTable(session, baseTableHandle, sysName)); } // The lake table is a real data table served by the paimon sibling, not the generic // partition_values function. @@ -248,7 +245,7 @@ public boolean isPartitionValuesSysTable(ConnectorSession session, public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { Connector owner = siblingOwner.apply(handle); if (owner != null) { - return siblingMetadata(session, owner).getTableSchema(session, handle); + return forward(session, owner, m -> m.getTableSchema(session, handle)); } FlussTableHandle flussHandle = (FlussTableHandle) handle; TableInfo info = tableInfo(session, flussHandle.toTablePath()); @@ -278,7 +275,7 @@ public Map getColumnHandles( ConnectorSession session, ConnectorTableHandle handle) { Connector owner = siblingOwner.apply(handle); if (owner != null) { - return siblingMetadata(session, owner).getColumnHandles(session, handle); + return forward(session, owner, m -> m.getColumnHandles(session, handle)); } FlussTableHandle flussHandle = (FlussTableHandle) handle; List columns = tableInfo(session, flussHandle.toTablePath()).getSchema().getColumns(); @@ -299,7 +296,7 @@ public String getTableComment(ConnectorSession session, String dbName, String ta public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { Connector owner = siblingOwner.apply(handle); if (owner != null) { - return siblingMetadata(session, owner).listPartitionNames(session, handle); + return forward(session, owner, m -> m.listPartitionNames(session, handle)); } List partitions = listPartitions(session, handle, Optional.empty()); List names = new ArrayList<>(partitions.size()); @@ -326,7 +323,7 @@ public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, Optional filter) { Connector owner = siblingOwner.apply(handle); if (owner != null) { - return siblingMetadata(session, owner).listPartitions(session, handle, filter); + return forward(session, owner, m -> m.listPartitions(session, handle, filter)); } FlussTableHandle flussHandle = (FlussTableHandle) handle; List partitionKeys = flussHandle.getPartitionKeys(); @@ -363,7 +360,7 @@ public Optional getTableStatistics( ConnectorSession session, ConnectorTableHandle handle) { Connector owner = siblingOwner.apply(handle); if (owner != null) { - return siblingMetadata(session, owner).getTableStatistics(session, handle); + return forward(session, owner, m -> m.getTableStatistics(session, handle)); } FlussTableHandle flussHandle = (FlussTableHandle) handle; long rowCount; diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java new file mode 100644 index 00000000000000..3c6553f7876dd6 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; + +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Runs a call into the lake sibling connector with the context classloader pinned to the sibling's own. + * + *

The engine pins the context classloader to the plugin whose object it is about to call + * ({@code PluginDrivenScanNode.onPluginClassLoader}), so everything this connector runs sees the FLUSS + * plugin's loader. That is right until this connector calls the sibling itself: the sibling's SDK is + * loaded child-first by ITS plugin, and the parts of it that discover implementations by + * {@code ServiceLoader} (catalog factories, file IO, file formats) look them up through the context + * classloader — which would be this plugin, where none of them exist. The failure is a + * {@code NoClassDefFoundError} or an empty factory list at the first lake table read, not at wiring time. + * + *

Every call that crosses into the sibling goes through here. The pin target is derived from an object + * the SIBLING created (its connector or its metadata), never named as a class: the sibling's types are + * invisible from this loader. + */ +final class LakeSibling { + + private LakeSibling() { + } + + /** + * Runs {@code body} with the context classloader set to the one that loaded {@code sibling}. + * + * @param sibling an object created by the sibling plugin (its {@code Connector} or its metadata) + * @param body the call to make into the sibling + */ + static T call(Object sibling, Supplier body) { + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(sibling.getClass().getClassLoader()); + try { + return body.get(); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + + /** + * Calls the sibling's metadata, pinned, through the per-statement funnel: the first call in a statement + * builds the metadata and every later one reuses that instance, mirroring what fe-core's own metadata + * funnel does for a plain connector. Building it is inside the pin too — that call already runs the + * sibling's code (it opens its catalog). + * + *

This is the only route to the sibling's metadata, so the pin and the memo cannot be forgotten by a + * new caller, and the metadata gateway and the scan planner cannot end up on two different instances + * (and therefore two different views of the lake table) within one statement. + * + *

The memo key carries the catalog id AND a role, because a fluss catalog runs two connectors + * (itself and the paimon sibling) under ONE catalog id: keying on the id alone would collapse them onto + * one metadata and misroute every call. + */ + static T forward(ConnectorSession session, Connector sibling, + Function call) { + String key = "metadata:" + session.getCatalogId() + ":lake"; + return call(sibling, () -> call.apply(session.getStatementScope() + .getOrCreateMetadata(key, () -> sibling.getMetadata(session)))); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java new file mode 100644 index 00000000000000..fa462837ac2bf0 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collections; + +/** + * The classloader pin every call into the lake sibling goes through. + * + *

Asserted against a classloader that is deliberately NOT the one the sibling was loaded by, because + * in a unit test both this class and the stand-in sibling come from the same loader: without a third, + * distinguishable one in play, "the pin happened" and "nothing happened" look identical. + */ +public class LakeSiblingTest { + + /** Stands in for the loader some other part of the engine had pinned before the call. */ + private ClassLoader callerLoader; + private ClassLoader previous; + + @BeforeEach + public void setUp() { + previous = Thread.currentThread().getContextClassLoader(); + callerLoader = new URLClassLoader(new URL[0], previous); + Thread.currentThread().setContextClassLoader(callerLoader); + } + + @AfterEach + public void tearDown() { + Thread.currentThread().setContextClassLoader(previous); + } + + /** + * The sibling's SDK discovers its catalogs, file systems and file formats through the context + * classloader. Left on the caller's, it looks for them in a plugin where none of them exist, and the + * failure surfaces at the first lake read rather than at wiring time. + */ + @Test + public void theCallRunsUnderTheSiblingsOwnClassLoader() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + ClassLoader seen = LakeSibling.call(sibling, + () -> Thread.currentThread().getContextClassLoader()); + + Assertions.assertSame(sibling.getClass().getClassLoader(), seen); + Assertions.assertNotSame(callerLoader, seen); + } + + /** What the caller had pinned is what the caller gets back; the pin is ours only for the call. */ + @Test + public void theCallersClassLoaderIsRestoredAfterwards() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + LakeSibling.call(sibling, () -> null); + + Assertions.assertSame(callerLoader, Thread.currentThread().getContextClassLoader()); + } + + /** + * Restored on the way out of a FAILING call too. A connector's errors are routine (a missing lake + * table, an unreachable warehouse), so leaking the pin on that path would leave every later call in + * the query resolving against the wrong plugin — with nothing to point at. + */ + @Test + public void theCallersClassLoaderIsRestoredWhenTheCallThrows() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + Assertions.assertThrows(IllegalStateException.class, + () -> LakeSibling.call(sibling, () -> { + throw new IllegalStateException("boom"); + })); + + Assertions.assertSame(callerLoader, Thread.currentThread().getContextClassLoader()); + } + + /** + * Building the sibling's metadata is itself a call into the sibling — it opens the lake catalog — so + * it has to be inside the pin, not before it. + */ + @Test + public void buildingTheSiblingsMetadataIsPinnedToo() { + LoaderProbe sibling = new LoaderProbe(); + + LakeSibling.forward(new FlussTestSession(1L, "q1"), sibling, + metadata -> metadata.getTableHandle(null, "db", "tbl")); + + Assertions.assertSame(sibling.getClass().getClassLoader(), sibling.loaderWhileBuilding); + } + + /** + * One metadata per statement, shared by every caller. Two instances would mean the scan planner and + * the metadata gateway could see two different versions of the same lake table within one statement. + */ + @Test + public void theSiblingsMetadataIsBuiltOncePerStatement() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + ConnectorSession session = new FlussTestSession(1L, "q1"); + + LakeSibling.forward(session, sibling, metadata -> metadata.getTableHandle(null, "db", "tbl")); + LakeSibling.forward(session, sibling, metadata -> metadata.getTableHandle(null, "db", "tbl")); + + Assertions.assertEquals(1, sibling.metadataBuilds); + } + + /** A sibling that records the context classloader in force while its metadata is being built. */ + private static final class LoaderProbe implements Connector { + + private ClassLoader loaderWhileBuilding; + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + loaderWhileBuilding = Thread.currentThread().getContextClassLoader(); + return new ConnectorMetadata() { + }; + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + throw new UnsupportedOperationException("not needed by this test"); + } + + @Override + public void close() { + } + } +} From d9b555914619c649e67c6d943ce43de62fe3656e Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 10:58:33 +0800 Subject: [PATCH 18/35] [feat](fluss) Read a tiered log table as its lake plus the log after it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A table that fluss tiers into a lake was refused outright: reading it as fluss-only would return just the rows tiering has not moved yet, which is a successful query with rows missing. It is now read as the union of the two halves — the lake at the snapshot fluss reports as readable, plus each bucket's log from exactly the offset that snapshot recorded, so the halves meet with no row read twice and none skipped. The lake half is not planned here. It is planned by the paimon sibling connector this catalog already builds for `tbl$lake`, on a handle pinned through the SPI's own snapshot hook, and its ranges are mixed into the same scan node — BE builds a reader per range, so the two kinds coexist. That keeps the plugin free of any paimon dependency (its zip is unchanged: connector jar, fluss-client, fe-foundation, jsr305) and gives the lake half paimon's native readers, deletion vectors and file cache for free. Nothing paimon-specific is named: the pin is a snapshot id with no connector options, and what fluss records as the lake snapshot IS the id the lake returned when tiering committed it. Three things the halves must agree on. The lake snapshot is read BEFORE the log's stopping offsets, so it can only be at or behind them; asked the other way round, a snapshot committed in between would cover rows past where the log half stops. The lake half is planned on the LAKE's own column handles — the sibling projects by its own handle type and ignores anything else, so fluss's handles would leave it reading every column, the three system columns tiering appends included. And the node properties of both halves travel in one map, since the engine populates the scan params once; the shared keys must already be equal, so a difference is raised rather than resolved by picking a side. A primary-key table tiered into a lake is still refused: its two halves have to be merged by key rather than concatenated. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../doris/connector/fluss/FlussConnector.java | 2 +- .../fluss/FlussScanPlanProvider.java | 354 +++++++++++++++--- .../connector/fluss/FlussSplitPlanTest.java | 327 +++++++++++++++- .../connector/fluss/FlussTestTables.java | 2 +- .../connector/fluss/RecordingLakeSibling.java | 145 ++++++- 5 files changed, 758 insertions(+), 72 deletions(-) diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 481fe54edd4ca4..3fa746617ca943 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -97,7 +97,7 @@ public ConnectorMetadata getMetadata(ConnectorSession session) { */ @Override public ConnectorScanPlanProvider getScanPlanProvider() { - return new FlussScanPlanProvider(adminOps(), properties); + return new FlussScanPlanProvider(adminOps(), properties, this::getOrCreateLakeSibling); } /** diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java index 8c66f6dda4df70..a77fef34b5601b 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -17,31 +17,38 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.api.Connector; import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.DorisConnectorException; import org.apache.doris.connector.api.handle.ConnectorColumnHandle; import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.api.pushdown.ConnectorExpression; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.api.scan.ConnectorScanRange; import org.apache.doris.connector.api.scan.ConnectorScanRequest; import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; import org.apache.fluss.client.table.scanner.log.LogScanner; import org.apache.fluss.exception.LakeTableSnapshotNotExistException; import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; /** * Turns a fluss table into the scan ranges that cover it. @@ -62,10 +69,17 @@ * SPI in this branch; the point is that fluss does not need one. A future change that takes a snapshot * lease during planning would.) * - *

What is NOT here yet: the union of a table's lake with its log. It is refused loudly rather than - * served as a partial answer — a datalake table read as fluss-only would silently return just the rows - * that have not been tiered away, which looks like a working query. - * {@code fluss.union_read.mode=disabled} is how a user asks for the fluss-only read on purpose. + *

A table that is tiered into a lake is read as the union of the two: the lake at the snapshot fluss + * says is readable, plus each bucket's log from where that snapshot ended. The lake half is not planned + * here — it is planned by the paimon sibling connector, pinned to that snapshot, and its ranges are mixed + * into the same scan node (BE builds a reader per range, so the two kinds coexist). That keeps this plugin + * free of any paimon dependency and gives the lake half paimon's native readers, deletion vectors and file + * cache for free. Refusing to serve such a table at all is the alternative to avoid: a datalake table read + * as fluss-only silently returns just the rows tiering has not moved yet, which looks like a working + * query. {@code fluss.union_read.mode=disabled} is how a user asks for that fluss-only read on purpose. + * + *

What is NOT here yet: the union of a PRIMARY-KEY table's lake with its log. That one cannot be a + * plain concatenation — the two halves have to be merged by key — so it is refused loudly. */ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { @@ -85,8 +99,12 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { static final String PROP_DB_NAME = "fluss.db_name"; static final String PROP_TABLE_NAME = "fluss.table_name"; + /** The only lake format that can be delegated today; fluss also defines iceberg / lance / hudi. */ + private static final String PAIMON_LAKE_FORMAT = "paimon"; + private final FlussAdminOps adminOps; private final Map catalogProperties; + private final Function, Connector> lakeSiblingFactory; /** * What the last {@link #planScan} produced, for the EXPLAIN line. Plain fields, not volatile: the @@ -97,50 +115,224 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { */ private int plannedLogRanges; private int plannedPkRanges; + private int plannedLakeSplits; private boolean plannedUnionRead; - public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties) { + /** + * This scan node's lake half, resolved at most once (see {@link #resolveUnionRead}). Same threading + * argument as the counters above; {@code unionResolved} distinguishes "not asked yet" from "asked, and + * this table has no lake half". + */ + private boolean unionResolved; + private UnionRead unionRead; + + public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties, + Function, Connector> lakeSiblingFactory) { this.adminOps = adminOps; this.catalogProperties = catalogProperties; + this.lakeSiblingFactory = lakeSiblingFactory; + } + + /** + * The lake half of a union read: the sibling connector that owns it, its scan planner, its table handle + * already pinned to {@link #snapshotId}, and where that snapshot left each bucket's log. + */ + private static final class UnionRead { + private final Connector sibling; + private final ConnectorScanPlanProvider siblingProvider; + private final ConnectorTableHandle pinnedLakeHandle; + private final long snapshotId; + private final Map logOffsets; + + private UnionRead(Connector sibling, ConnectorScanPlanProvider siblingProvider, + ConnectorTableHandle pinnedLakeHandle, long snapshotId, + Map logOffsets) { + this.sibling = sibling; + this.siblingProvider = siblingProvider; + this.pinnedLakeHandle = pinnedLakeHandle; + this.snapshotId = snapshotId; + this.logOffsets = logOffsets; + } } @Override public List planScan(ConnectorSession session, ConnectorScanRequest request) { FlussTableHandle handle = (FlussTableHandle) request.getTableHandle(); - FlussConnectorProperties.UnionReadMode mode = - FlussConnectorProperties.unionReadMode(catalogProperties); - rejectWhatIsNotImplemented(handle, mode); + UnionRead union = resolveUnionRead(session, handle); List buckets = allBuckets(handle.getBucketCount()); List ranges = new ArrayList<>(); + // The lake half first, so its ranges lead the list the way they lead the table's history. It is + // planned once for the whole table: the sibling prunes partitions from the pushed-down filter, not + // from the engine's pruned partition list (which it does not consume). + if (union != null) { + ranges.addAll(planLakeRanges(session, union, request)); + } + plannedLakeSplits = ranges.size(); + if (handle.isPartitioned()) { for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { // fluss's own partition name ("20260101$cn"), not the Doris one: this is a fluss API. - appendPartitionRanges(ranges, handle, + appendPartitionRanges(ranges, handle, union, FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), buckets, partition.getPartitionName()); } } else { - appendPartitionRanges(ranges, handle, FlussScanRange.Partition.NONE, buckets, null); + appendPartitionRanges(ranges, handle, union, FlussScanRange.Partition.NONE, buckets, null); } plannedLogRanges = count(ranges, FlussScanRange.RangeType.LOG); plannedPkRanges = count(ranges, FlussScanRange.RangeType.PK_FULL); - plannedUnionRead = false; + plannedUnionRead = union != null; return ranges; } + /** + * The lake half, planned by the sibling connector on the handle already pinned to the readable snapshot. + * + *

Only the filter is carried over. The row limit is deliberately dropped — applying it to one half of + * a union would silently drop rows from the other — and so is the {@code COUNT(*)} signal, because a + * per-range row count is not this table's count once the log half is added. The engine's pruned + * partition list is not passed either: the sibling does not consume it (it re-plans from the filter), + * and pretending otherwise would hide that the two halves prune by different means. + */ + private List planLakeRanges(ConnectorSession session, UnionRead union, + ConnectorScanRequest request) { + List lakeColumns = + lakeColumns(session, union, request.getColumns(), request.getTableHandle()); + ConnectorScanRequest lakeRequest = + ConnectorScanRequest.builder(union.pinnedLakeHandle, lakeColumns) + .filter(request.getFilter()) + .build(); + return LakeSibling.call(union.sibling, + () -> union.siblingProvider.planScan(session, lakeRequest)); + } + + /** + * The sibling's own column handles for the columns this scan reads. + * + *

Needed because the sibling projects by ITS handle type and silently ignores anything else: handing + * it fluss's handles would leave it with no projection at all. The lake table's columns are this table's + * columns plus the three fluss system columns appended at the end ({@code __bucket} / {@code __offset} / + * {@code __timestamp}), so every column asked for here exists there under the same name and the extra + * three are simply never asked for. A column that is missing is a real mismatch between the two schemas + * — the lake table was not created by this fluss table's tiering — and fails loud rather than reading a + * silently narrower row. + */ + private List lakeColumns(ConnectorSession session, UnionRead union, + List columns, ConnectorTableHandle handle) { + Map lakeHandles = LakeSibling.forward(session, union.sibling, + metadata -> metadata.getColumnHandles(session, union.pinnedLakeHandle)); + List mapped = new ArrayList<>(columns.size()); + for (ConnectorColumnHandle column : columns) { + String name = ((FlussColumnHandle) column).getName(); + ConnectorColumnHandle lakeHandle = lakeHandles.get(name); + if (lakeHandle == null) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + throw new DorisConnectorException("Column '" + name + "' of fluss table '" + + flussHandle.getDatabaseName() + "." + flussHandle.getTableName() + + "' does not exist in its lake table, so the two cannot be read as one"); + } + mapped.add(lakeHandle); + } + return mapped; + } + + /** + * This scan node's lake half, or null when the table is read from fluss alone. Resolved at most once per + * scan node because two entry points need it — {@link #planScan} and {@link #getScanNodeProperties} — + * and the engine may call either first. + * + *

Resolving it once is also what keeps the two halves from overlapping. The lake snapshot is read + * HERE, before {@link #planScan} asks fluss where each bucket's log currently ends; log offsets only + * move forward, so the snapshot can only be at or behind those stopping offsets. Read the other way + * round, a snapshot committed in between would cover rows past the point the log half stops at, and the + * bucket's log range would start after it ends. + */ + private UnionRead resolveUnionRead(ConnectorSession session, FlussTableHandle handle) { + if (unionResolved) { + return unionRead; + } + unionResolved = true; + unionRead = resolveUnionReadUncached(session, handle); + return unionRead; + } + + private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableHandle handle) { + FlussConnectorProperties.UnionReadMode mode = + FlussConnectorProperties.unionReadMode(catalogProperties); + if (!handle.isDataLakeEnabled() || mode == FlussConnectorProperties.UnionReadMode.DISABLED) { + // Not a lake table, or the user asked for the fluss-only read explicitly. + return null; + } + LakeSnapshot snapshot; + try { + snapshot = adminOps.getReadableLakeSnapshot(handle.toTablePath()); + } catch (LakeTableSnapshotNotExistException e) { + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' has no readable lake snapshot yet, and '" + + FlussConnectorProperties.UNION_READ_MODE + "=required' forbids falling back to " + + "a fluss-only read. Wait for the tiering service to commit, or set the property " + + "to auto or disabled.", e); + } + // Nothing is in the lake, so the log holds everything: the fluss-only read is the whole table. + return null; + } + if (handle.hasPrimaryKey()) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' is a primary-key table tiered into a lake. Reading it" + + " requires merging the lake with the change log by key, which is not supported yet." + + " Set '" + FlussConnectorProperties.UNION_READ_MODE + "=disabled' to read only what" + + " the fluss log still holds, which is NOT the whole table."); + } + String lakeFormat = handle.getDataLakeFormat(); + if (lakeFormat == null || !PAIMON_LAKE_FORMAT.equalsIgnoreCase(lakeFormat)) { + throw new DorisConnectorException("Cannot read table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "': its table.datalake.format is '" + lakeFormat + + "', and the fluss connector currently supports only '" + PAIMON_LAKE_FORMAT + "'"); + } + + Connector sibling = lakeSiblingFactory.apply( + PaimonSiblingProperties.synthesize(handle.getProperties())); + ConnectorTableHandle lakeHandle = LakeSibling.forward(session, sibling, + metadata -> metadata.getTableHandle( + session, handle.getDatabaseName(), handle.getTableName())) + .orElseThrow(() -> new DorisConnectorException("Fluss reports a readable lake snapshot for '" + + handle.getDatabaseName() + "." + handle.getTableName() + "' but its lake table" + + " does not exist. The lake warehouse and the fluss cluster disagree; check the" + + " table's table.datalake.* settings")); + // The pin is expressed in the SPI's own terms — a snapshot id and no connector options — so the + // sibling translates it into whatever its SDK calls a snapshot. Nothing paimon-specific is named + // here. The id needs no mapping either: what fluss records as the lake snapshot IS the id the lake + // returned when tiering committed it. + ConnectorMvccSnapshot pin = ConnectorMvccSnapshot.builder() + .snapshotId(snapshot.getSnapshotId()) + .build(); + ConnectorTableHandle pinnedHandle = LakeSibling.forward(session, sibling, + metadata -> metadata.applySnapshot(session, lakeHandle, pin)); + ConnectorScanPlanProvider siblingProvider = LakeSibling.call(sibling, + () -> sibling.getScanPlanProvider(pinnedHandle)); + return new UnionRead(sibling, siblingProvider, pinnedHandle, snapshot.getSnapshotId(), + snapshot.getTableBucketsOffset()); + } + /** * The ranges covering one partition of a table, or the whole of an unpartitioned one * ({@code flussPartitionName} is null, which is also how the two admin overloads are told apart). */ private void appendPartitionRanges(List ranges, FlussTableHandle handle, - FlussScanRange.Partition partition, List buckets, String flussPartitionName) { + UnionRead union, FlussScanRange.Partition partition, List buckets, + String flussPartitionName) { TablePath tablePath = handle.toTablePath(); if (!handle.hasPrimaryKey()) { - appendLogRanges(ranges, partition, buckets, - latestOffsets(tablePath, flussPartitionName, buckets)); + Map stopping = latestOffsets(tablePath, flussPartitionName, buckets); + if (union == null) { + appendLogRanges(ranges, partition, buckets, stopping); + } else { + appendUnionLogRanges(ranges, handle, union, partition, buckets, stopping); + } return; } // Snapshots BEFORE offsets, and the order is load-bearing. A snapshot committed between the two @@ -166,38 +358,6 @@ private Map latestOffsets(TablePath tablePath, String flussPartit : adminOps.listOffsets(tablePath, flussPartitionName, buckets, new OffsetSpec.LatestSpec()); } - /** - * Refuses the reads this connector cannot serve yet, naming what would have to change. Serving them - * partially is the failure mode to avoid: a datalake table planned as fluss-only returns whatever - * the log still holds and drops everything tiering has already moved into the lake, which is a - * successful query with missing rows. - */ - private void rejectWhatIsNotImplemented(FlussTableHandle handle, - FlussConnectorProperties.UnionReadMode mode) { - if (!handle.isDataLakeEnabled() || mode == FlussConnectorProperties.UnionReadMode.DISABLED) { - // Not a lake table, or the user asked for the fluss-only read explicitly. - return; - } - try { - adminOps.getReadableLakeSnapshot(handle.toTablePath()); - } catch (LakeTableSnapshotNotExistException e) { - if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { - throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." - + handle.getTableName() + "' has no readable lake snapshot yet, and '" - + FlussConnectorProperties.UNION_READ_MODE + "=required' forbids falling back to " - + "a fluss-only read. Wait for the tiering service to commit, or set the property " - + "to auto or disabled.", e); - } - // Nothing is in the lake, so the log holds everything: the fluss-only read is the whole table. - return; - } - throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." - + handle.getTableName() + "' is tiered into a lake and reading it requires combining the " - + "lake with the fluss log, which is not supported yet. Set '" - + FlussConnectorProperties.UNION_READ_MODE + "=disabled' to read only what the fluss log " - + "still holds, which is NOT the whole table."); - } - /** * The partitions to scan: those the engine's pruning left, or all of them when it pruned nothing. * A pruned name that fluss no longer lists is simply absent from the result — the partition was @@ -236,6 +396,35 @@ private static void appendLogRanges(List ranges, } } + /** + * The log half of a union read: each bucket from where the lake snapshot left off, up to where planning + * saw the log. The snapshot's offset is exclusive — it is the first offset NOT in the lake — so the two + * halves meet exactly, with no row read twice and none skipped. + * + *

A bucket the snapshot does not mention has never been tiered, so its log is read from the earliest + * offset fluss still holds; a bucket whose snapshot offset has caught up with the stopping offset has + * nothing left outside the lake and yields no range at all. + */ + private static void appendUnionLogRanges(List ranges, FlussTableHandle handle, + UnionRead union, FlussScanRange.Partition partition, List buckets, + Map stopping) { + for (int bucket : buckets) { + Long stop = stopping.get(bucket); + if (stop == null || stop <= 0) { + continue; + } + TableBucket tableBucket = partition.isPartitioned() + ? new TableBucket(handle.getTableId(), partition.getId(), bucket) + : new TableBucket(handle.getTableId(), bucket); + Long lakeEnd = union.logOffsets.get(tableBucket); + if (lakeEnd == null) { + ranges.add(FlussScanRange.log(partition, bucket, LogScanner.EARLIEST_OFFSET, stop)); + } else if (lakeEnd < stop) { + ranges.add(FlussScanRange.log(partition, bucket, lakeEnd, stop)); + } + } + } + /** * One range per bucket that holds anything: its latest kv snapshot, plus the change log from where * that snapshot ended up to where planning saw the log. A bucket fluss has never snapshotted gets @@ -259,10 +448,15 @@ private static void appendPkRanges(List ranges, } } + /** + * How many of {@code ranges} are fluss ranges of this kind. The type test is not defensive: on a union + * read the list also holds the lake half's ranges, which are the sibling's own type and would fail a + * cast. + */ private static int count(List ranges, FlussScanRange.RangeType rangeType) { int found = 0; for (ConnectorScanRange range : ranges) { - if (((FlussScanRange) range).getRangeType() == rangeType) { + if (range instanceof FlussScanRange && ((FlussScanRange) range).getRangeType() == rangeType) { found++; } } @@ -296,20 +490,80 @@ public Map getScanNodeProperties(ConnectorSession session, Conne props.put(PROP_TABLE_NAME, flussHandle.getTableName()); FlussConnectorProperties.toFlussClientConfig(catalogProperties) .forEach((key, value) -> props.put(PROP_CLIENT_PREFIX + key, value)); + + UnionRead union = resolveUnionRead(session, flussHandle); + if (union != null) { + List lakeColumns = lakeColumns(session, union, columns, handle); + mergeLakeProperties(props, LakeSibling.call(union.sibling, + () -> union.siblingProvider.getScanNodeProperties( + session, union.pinnedLakeHandle, lakeColumns, filter))); + } return props; } + /** + * Folds the lake half's node properties into this scan node's, which is what makes one node able to + * serve both kinds of range: the engine calls {@code populateScanLevelParams} once, with one map, and + * the sibling reads its own entries back out of it. + * + *

The two connectors share three keys, and all three must already agree, so a difference is a real + * mismatch and is raised rather than resolved by picking a side. {@code path_partition_keys} is the one + * that matters: the split between file columns and partition columns is decided ONCE for the node, so + * two halves that disagree about which columns come from the range would read different columns from + * the same tuple. They cannot legitimately disagree — a fluss table's lake table is created with its + * partition keys — which is exactly why a disagreement means something is wrong upstream. + */ + private static void mergeLakeProperties(Map props, Map lakeProps) { + for (Map.Entry entry : lakeProps.entrySet()) { + String existing = props.get(entry.getKey()); + if (existing == null) { + props.put(entry.getKey(), entry.getValue()); + } else if (!existing.equals(entry.getValue())) { + throw new DorisConnectorException("The fluss table and its lake table disagree about scan" + + " property '" + entry.getKey() + "' ('" + existing + "' vs '" + entry.getValue() + + "'), so they cannot be read as one"); + } + } + } + @Override public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { Map beProperties = new LinkedHashMap<>(); for (Map.Entry entry : nodeProperties.entrySet()) { - // Prefix-gated rather than copied wholesale: the map also carries the engine's own keys and - // the synthetic ones it injects for EXPLAIN, none of which mean anything to the scanner. + // Prefix-gated rather than copied wholesale: the map also carries the engine's own keys, the + // synthetic ones it injects for EXPLAIN and (on a union read) the lake half's, none of which + // mean anything to the scanner. if (entry.getKey().startsWith(BE_PROPERTY_PREFIX)) { beProperties.put(entry.getKey(), entry.getValue()); } } params.setFlussProperties(beProperties); + + // The lake half's turn at the same params. Its ranges are useless without it — the paimon reader + // fails outright on a missing serialized table — and only the sibling knows which of the merged + // entries are its own. + UnionRead union = unionRead; + if (union != null) { + LakeSibling.call(union.sibling, () -> { + union.siblingProvider.populateScanLevelParams(params, nodeProperties); + return null; + }); + } + } + + /** + * Forwarded so the VERBOSE EXPLAIN counts the lake half's merge-on-read delete files. A fluss range + * carries none, and the sibling reads them off its own per-range descriptor, so asking it about every + * range is both harmless and the only way to ask without knowing which range is whose. + */ + @Override + public List getDeleteFiles(TTableFormatFileDesc tableFormatParams) { + UnionRead union = unionRead; + if (union == null) { + return Collections.emptyList(); + } + return LakeSibling.call(union.sibling, + () -> union.siblingProvider.getDeleteFiles(tableFormatParams)); } /** @@ -323,7 +577,7 @@ public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { output.append(prefix) .append("flussScan: unionRead=").append(plannedUnionRead ? "yes" : "no") - .append(", lakeSplits=0") + .append(", lakeSplits=").append(plannedLakeSplits) .append(", logRanges=").append(plannedLogRanges) .append(", pkRanges=").append(plannedPkRanges) .append(", mode=") diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java index 386ee8a9f2e53f..9fc29f5d760f22 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -17,9 +17,11 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.api.Connector; import org.apache.doris.connector.api.ConnectorPartitionInfo; import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.scan.ConnectorScanRange; import org.apache.doris.connector.api.scan.ConnectorScanRequest; @@ -30,6 +32,7 @@ import org.apache.fluss.client.metadata.LakeSnapshot; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Assertions; @@ -62,10 +65,20 @@ public class FlussSplitPlanTest { private RecordingFlussAdminOps adminOps; private ConnectorSession session; + /** + * The lake sibling, built only when a test asks for it. A test that never registers a lake table must + * never reach it, and {@link #lakeSibling} fails loud if it does — a fluss-only plan that quietly + * consulted the lake would still look correct in every other assertion. + */ + private RecordingLakeSibling sibling; + private boolean siblingExpected; + @BeforeEach public void setUp() { adminOps = new RecordingFlussAdminOps(); session = new FlussTestSession(1L, "q1"); + sibling = null; + siblingExpected = false; } // ---------------------------------------------------------------- unpartitioned log table @@ -352,6 +365,8 @@ public void tieredPrimaryKeyTableIsRefusedUntilTheUnionReadExists() { .buckets(2, "id") .property("table.datalake.enabled", "true") .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") .build()); adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); @@ -360,23 +375,223 @@ public void tieredPrimaryKeyTableIsRefusedUntilTheUnionReadExists() { Assertions.assertTrue(e.getMessage().contains("not supported yet"), e.getMessage()); } + + // ---------------------------------------------------------------- union read: lake + log + /** - * A lake table read as fluss-only returns only what the log still holds, dropping everything tiering - * has moved into the lake — a query that succeeds with missing rows. Until the union read exists, - * the refusal is the correct answer. + * The two halves have to MEET: the lake holds everything up to the offset its snapshot recorded, and + * the log range starts at exactly that offset. One off in either direction is a wrong answer that no + * row count catches — reading the log from the earliest offset instead duplicates every tiered row, + * and starting one later drops one. */ @Test - public void tieredLakeTableIsRefusedUntilTheUnionReadExists() { + public void theLogIsReadFromWhereTheLakeSnapshotLeftOff() { registerLakeTable(2); - adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); + lakeSnapshotAt(7L, offsets(5L, 3L)); + latestOffsets(null, 9L, 3L); + lakeRanges(2); + + List ranges = plan(LOG_TABLE, catalog()); + + // Two lake ranges, then the one bucket whose log went past the snapshot. Bucket 1's log has not + // moved since it was tiered, so everything it holds is already in the lake and it yields nothing. + Assertions.assertEquals(3, ranges.size()); + assertLogRange(ranges.get(2), 0, 5L, 9L); + } + + /** + * A bucket the lake snapshot never mentions has never been tiered, so nothing of it is in the lake + * and its whole log is still the truth. Starting it at the snapshot's (absent) offset would be the + * bug: there is no offset to start at, and treating that as zero rows would drop the bucket. + */ + @Test + public void bucketTheLakeNeverSawIsReadFromTheEarliestOffset() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(4L, null)); + latestOffsets(null, 6L, 8L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertLogRange(ranges.get(0), 0, 4L, 6L); + assertLogRange(ranges.get(1), 1, -2L, 8L); + } + + /** A bucket whose log has not moved past the lake needs no log range at all. */ + @Test + public void bucketTheLakeHasCaughtUpWithYieldsNoLogRange() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(6L, 6L)); + latestOffsets(null, 6L, 6L); + + Assertions.assertTrue(plan(LOG_TABLE, catalog()).isEmpty()); + } + + /** + * The lake is pinned to the snapshot FLUSS says is readable, not to whatever the lake calls latest. + * Unpinned, the lake half would drift ahead of the offsets the log half stops at and the rows in + * between would be read twice. + */ + @Test + public void theLakeIsPinnedToTheSnapshotFlussReportsAsReadable() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(2L)); + latestOffsets(null, 5L); + + plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(7L, sibling.plannedHandle.pinnedSnapshotId); + // The pin is expressed in the SPI's terms only: a snapshot id and no connector-specific options. + Assertions.assertTrue(sibling.calls.contains("applySnapshot:7:{}"), sibling.calls.toString()); + } + + /** + * The lake snapshot must be read BEFORE the log's stopping offsets. Log offsets only move forward, so + * asking in this order keeps the snapshot at or behind where the log half stops; asked the other way + * round, a snapshot committed in between would cover rows past that point and the bucket's log range + * would start after it ended. + */ + @Test + public void theLakeSnapshotIsReadBeforeTheStoppingOffsets() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(2L)); + latestOffsets(null, 5L); + + plan(LOG_TABLE, catalog()); + + int snapshotCall = indexOfCall("getReadableLakeSnapshot"); + int offsetsCall = indexOfCall("listOffsets"); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < offsetsCall, adminOps.calls.toString()); + } + + /** + * The lake half is planned on the LAKE's own column handles. The sibling projects by its own handle + * type and ignores anything else, so handing it fluss's handles would leave it reading every column — + * including the three system columns tiering appends — and no assertion about rows would notice. + */ + @Test + public void theLakeHalfIsPlannedOnTheLakeTablesOwnColumnHandles() { + registerLakeTable(1); + lakeTableColumns("id", "__bucket", "__offset", "__timestamp"); + lakeSnapshotAt(7L, offsets(0L)); + latestOffsets(null, 5L); + + new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).planScan(session, + ConnectorScanRequest.builder(handle(LOG_TABLE), + Collections.singletonList(new FlussColumnHandle("id", 0))).build()); + + Assertions.assertEquals( + Collections.singletonList(new RecordingLakeSibling.LakeColumn("id")), + sibling.plannedColumns); + } + + /** A column the lake table does not have means the two schemas are not the same table's. */ + @Test + public void columnMissingFromTheLakeTableIsRefused() { + registerLakeTable(1); + lakeTableColumns("id"); + lakeSnapshotAt(7L, offsets(0L)); DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, - () -> plan(LOG_TABLE, catalog())); - Assertions.assertTrue(e.getMessage().contains("not supported yet"), e.getMessage()); - Assertions.assertTrue(e.getMessage().contains(FlussConnectorProperties.UNION_READ_MODE), + () -> new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).planScan(session, + ConnectorScanRequest.builder(handle(LOG_TABLE), + Collections.singletonList(new FlussColumnHandle("gone", 1))).build())); + Assertions.assertTrue(e.getMessage().contains("does not exist in its lake table"), e.getMessage()); + } + + /** + * One scan node, one property map: the engine calls {@code populateScanLevelParams} once, so the lake + * half's entries have to travel in the same map or its ranges arrive at BE unreadable. + */ + @Test + public void nodePropertiesCarryBothHalves() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); // built up-front so the node properties can be set before they are asked for + sibling.lakeNodeProperties = Collections.singletonMap("paimon.serialized_table", "encoded"); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertEquals("db", props.get("fluss.db_name")); + Assertions.assertEquals("encoded", props.get("paimon.serialized_table")); + } + + /** The lake half gets its turn at the same thrift params, and only it knows which entries are its. */ + @Test + public void scanLevelParamsAreOfferedToTheLakeHalfToo() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); + sibling.lakeNodeProperties = Collections.singletonMap("paimon.serialized_table", "encoded"); + Map catalog = catalog(); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); + Map props = provider.getScanNodeProperties( + session, handle(LOG_TABLE), Collections.emptyList(), Optional.empty()); + + TFileScanRangeParams params = new TFileScanRangeParams(); + provider.populateScanLevelParams(params, props); + + // fluss took only its own keys; the sibling saw the whole map, its own entries included. + Assertions.assertFalse(params.getFlussProperties().containsKey("paimon.serialized_table")); + Assertions.assertEquals("encoded", + sibling.populatedNodeProperties.get("paimon.serialized_table")); + } + + /** + * The split between file columns and partition columns is decided ONCE for the scan node, so two + * halves that disagree about it would read different columns out of the same tuple. They cannot + * legitimately disagree, which is why a disagreement is raised rather than resolved by picking one. + */ + @Test + public void halvesThatDisagreeAboutTheSharedPropertiesAreRefused() { + registerPartitionedLakeTable(1, "20260101"); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); + sibling.lakeNodeProperties = + Collections.singletonMap(ScanNodePropertyKeys.PATH_PARTITION_KEYS, "not_dt"); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> nodeProperties(LOG_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains(ScanNodePropertyKeys.PATH_PARTITION_KEYS), e.getMessage()); } + /** A partitioned table's lake offsets are per (partition, bucket), not per bucket. */ + @Test + public void partitionedTablesLogIsResumedPerPartitionAndBucket() { + registerPartitionedLakeTable(1, "20260101", "20260102"); + Map lakeOffsets = new HashMap<>(); + lakeOffsets.put(new TableBucket(FlussTestTables.TABLE_ID, 100L, 0), 4L); + lakeOffsets.put(new TableBucket(FlussTestTables.TABLE_ID, 101L, 0), 1L); + lakeSnapshotAt(7L, lakeOffsets); + latestOffsets("20260101", 9L); + latestOffsets("20260102", 6L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertLogRange(ranges.get(0), 0, 4L, 9L); + assertLogRange(ranges.get(1), 0, 1L, 6L); + } + + /** The line a union-read regression test reads to know the lake was actually part of the answer. */ + @Test + public void explainReportsTheUnionAndTheSizeOfEachHalf() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(1L, 9L)); + latestOffsets(null, 5L, 9L); + lakeRanges(3); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals( + "flussScan: unionRead=yes, lakeSplits=3, logRanges=1, pkRanges=0, mode=auto\n", + output.toString()); + } + /** Nothing in the lake means the log holds the whole table, so the fluss-only read IS complete. */ @Test public void lakeTableThatHasNeverTieredFallsBackToTheLogAlone() { @@ -420,7 +635,7 @@ public void scanLevelParamsCarryTheClientConfigAndTableIdentity() { catalog.put("fluss.client.writer.batch-size", "2mb"); TFileScanRangeParams params = new TFileScanRangeParams(); - FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); provider.populateScanLevelParams(params, nodeProperties(LOG_TABLE, catalog)); Map expected = new LinkedHashMap<>(); @@ -439,7 +654,7 @@ public void scanLevelParamsDropTheEngineOnlyKeys() { nodeProps.put(ScanNodePropertyKeys.SYNTHETIC_TOTAL_READ_SPLITS, "3"); TFileScanRangeParams params = new TFileScanRangeParams(); - new FlussScanPlanProvider(adminOps, catalog()).populateScanLevelParams(params, nodeProps); + new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).populateScanLevelParams(params, nodeProps); Assertions.assertFalse(params.getFlussProperties() .containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); @@ -455,7 +670,7 @@ public void scanLevelParamsDropTheEngineOnlyKeys() { public void explainReportsHowTheScanWasActuallyPlanned() { registerLogTable(LOG_TABLE, 3); latestOffsets(null, 1L, 0L, 5L); - FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog()); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); StringBuilder output = new StringBuilder(); @@ -476,7 +691,7 @@ public void explainCountsPrimaryKeyRangesApartFromLogRanges() { registerPkTable(PK_TABLE, 3); kvSnapshots(null, new long[] {1L, NO_SNAPSHOT, NO_SNAPSHOT}, new long[] {10L, 0L, 0L}); latestOffsets(null, 12L, 4L, 0L); - FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog()); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); StringBuilder output = new StringBuilder(); @@ -492,7 +707,7 @@ public void explainReportsTheConfiguredMode() { registerLogTable(LOG_TABLE, 1); latestOffsets(null, 1L); Map catalog = catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"); - FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); StringBuilder output = new StringBuilder(); @@ -533,12 +748,12 @@ private List plan(TablePath tablePath, Map c private List plan(TablePath tablePath, Map catalogProperties, List requiredPartitions) { - return new FlussScanPlanProvider(adminOps, catalogProperties) + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling) .planScan(session, request(handle(tablePath), requiredPartitions)); } private Map nodeProperties(TablePath tablePath, Map catalogProperties) { - return new FlussScanPlanProvider(adminOps, catalogProperties).getScanNodeProperties( + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling).getScanNodeProperties( session, handle(tablePath), Collections.emptyList(), Optional.empty()); } @@ -578,7 +793,10 @@ private void registerLakeTable(int buckets) { .buckets(buckets) .property("table.datalake.enabled", "true") .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") .build()); + siblingExpected = true; } private void registerPkTable(TablePath tablePath, int buckets) { @@ -669,4 +887,83 @@ private static void assertPartition(ConnectorScanRange range, String partitionNa assertLogRange(range, bucket, -2L, stop); } + // ---------------------------------------------------------------- union-read fixtures + + /** + * The lake sibling this catalog would build. Fails loud when a test that registered no lake table + * reaches it: a fluss-only plan that quietly consulted the lake still passes every other assertion. + */ + private Connector lakeSibling(Map siblingProperties) { + if (!siblingExpected) { + throw new AssertionError("no lake sibling is expected in this test"); + } + if (sibling == null) { + sibling = new RecordingLakeSibling(siblingProperties); + } + return sibling; + } + + /** The sibling, built now so a test can set what it answers before planning asks. */ + private RecordingLakeSibling sibling() { + lakeSibling(Collections.emptyMap()); + return sibling; + } + + /** The readable lake snapshot fluss reports, with the log offset it recorded for each bucket. */ + private void lakeSnapshotAt(long snapshotId, Map bucketOffsets) { + adminOps.readableLakeSnapshot = new LakeSnapshot(snapshotId, bucketOffsets); + } + + /** + * Lake offsets for buckets 0..n-1 of an unpartitioned table; a {@code null} entry is a bucket the + * snapshot does not mention at all (never tiered), which is not the same as offset 0. + */ + private static Map offsets(Long... byBucket) { + Map offsets = new HashMap<>(); + for (int bucket = 0; bucket < byBucket.length; bucket++) { + if (byBucket[bucket] != null) { + offsets.put(new TableBucket(FlussTestTables.TABLE_ID, bucket), byBucket[bucket]); + } + } + return offsets; + } + + /** {@code count} ranges for the sibling's scan planner to return as the lake half. */ + private void lakeRanges(int count) { + List ranges = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + ranges.add(new RecordingLakeSibling.LakeRange()); + } + sibling().lakeRanges = ranges; + } + + /** The columns the lake table reports, in order. */ + private void lakeTableColumns(String... names) { + Map columns = new LinkedHashMap<>(); + for (String name : names) { + columns.put(name, new RecordingLakeSibling.LakeColumn(name)); + } + sibling().lakeColumns = columns; + } + + /** A lake-enabled log table partitioned by {@code dt}, with partition ids 100, 101, ... in order. */ + private void registerPartitionedLakeTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionedBy("dt") + .buckets(buckets) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(LOG_TABLE, partitions); + siblingExpected = true; + } } diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java index 2a930096d58742..0eb2271a641820 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java @@ -40,7 +40,7 @@ final class FlussTestTables { /** Ids a fixture does not care about; a real cluster assigns its own. */ - private static final long TABLE_ID = 1L; + static final long TABLE_ID = 1L; private static final int SCHEMA_ID = 1; private FlussTestTables() { diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java index a15d8a3013c5df..63f4e631d2b111 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java @@ -27,8 +27,12 @@ import org.apache.doris.connector.api.ConnectorType; import org.apache.doris.connector.api.handle.ConnectorColumnHandle; import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.api.pushdown.ConnectorExpression; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.thrift.TFileScanRangeParams; import java.util.ArrayList; import java.util.Collections; @@ -48,16 +52,29 @@ */ final class RecordingLakeSibling implements Connector { - /** The sibling's own handle type: the analogue of {@code PaimonTableHandle}. */ + /** Fixture shorthand: this handle has not been pinned to a snapshot. */ + static final long UNPINNED = Long.MIN_VALUE; + + /** + * The sibling's own handle type: the analogue of {@code PaimonTableHandle}. It carries the pin the + * same way a real one does — as a NEW handle returned by {@code applySnapshot} — so a test can tell a + * scan that was planned on the pinned handle from one planned on the raw one. + */ static final class Handle implements ConnectorTableHandle { private static final long serialVersionUID = 1L; final String dbName; final String tableName; + final long pinnedSnapshotId; Handle(String dbName, String tableName) { + this(dbName, tableName, UNPINNED); + } + + Handle(String dbName, String tableName, long pinnedSnapshotId) { this.dbName = dbName; this.tableName = tableName; + this.pinnedSnapshotId = pinnedSnapshotId; } } @@ -69,7 +86,7 @@ static final class Handle implements ConnectorTableHandle { static final long ROW_COUNT = 4242L; /** The scan plan provider this sibling hands out; identity is what the routing test asserts. */ - final ConnectorScanPlanProvider scanPlanProvider = (session, request) -> Collections.emptyList(); + final ConnectorScanPlanProvider scanPlanProvider = new ScanPlanProvider(); final Map properties; final List calls = new ArrayList<>(); @@ -77,6 +94,107 @@ static final class Handle implements ConnectorTableHandle { /** When false, the sibling reports the lake table as absent (nothing tiered to it yet). */ boolean lakeTableExists = true; + /** + * The columns the lake table reports, by name. Defaults to the single lake-only column the gateway + * tests assert on; a union test replaces it with the fluss table's columns plus the three system + * columns tiering appends, which is what a real lake table looks like. + */ + Map lakeColumns = defaultLakeColumns(); + + /** The ranges this sibling's scan planner returns, so a union test can recognize the lake half. */ + List lakeRanges = Collections.emptyList(); + + /** The node properties this sibling's scan planner reports, to be merged into the scan node's. */ + Map lakeNodeProperties = Collections.emptyMap(); + + /** The handle the scan planner was last asked to plan; proves WHICH handle reached the lake half. */ + Handle plannedHandle; + + /** The columns the scan planner was last asked to plan, in order. */ + List plannedColumns = Collections.emptyList(); + + /** The node properties last handed to {@code populateScanLevelParams}. */ + Map populatedNodeProperties; + + private static Map defaultLakeColumns() { + Map handles = new LinkedHashMap<>(); + handles.put(COLUMN_NAME, new LakeColumn(COLUMN_NAME)); + return handles; + } + + /** The sibling's own column-handle type, which fluss can only pass through, never build. */ + static final class LakeColumn implements ConnectorColumnHandle { + private static final long serialVersionUID = 1L; + + final String name; + + LakeColumn(String name) { + this.name = name; + } + + @Override + public boolean equals(Object other) { + return other instanceof LakeColumn && name.equals(((LakeColumn) other).name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "LakeColumn{" + name + "}"; + } + } + + /** + * The sibling's own range type. Deliberately NOT a {@link FlussScanRange}: on a union read both kinds + * share one list, and a fixture that handed back fluss ranges would hide every place that assumes the + * list is homogeneous. + */ + static final class LakeRange implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + + @Override + public String getTableFormatType() { + return "paimon"; + } + + @Override + public Map getProperties() { + return Collections.emptyMap(); + } + } + + /** Records what the lake half was asked to plan and answers with this sibling's canned values. */ + private final class ScanPlanProvider implements ConnectorScanPlanProvider { + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + calls.add("planScan"); + plannedHandle = (Handle) request.getTableHandle(); + plannedColumns = new ArrayList<>(request.getColumns()); + return lakeRanges; + } + + @Override + public Map getScanNodeProperties(ConnectorSession session, + ConnectorTableHandle handle, List columns, + Optional filter) { + calls.add("getScanNodeProperties"); + plannedColumns = new ArrayList<>(columns); + return lakeNodeProperties; + } + + @Override + public void populateScanLevelParams(TFileScanRangeParams params, + Map nodeProperties) { + calls.add("populateScanLevelParams"); + populatedNodeProperties = new LinkedHashMap<>(nodeProperties); + } + } + /** How many metadata instances this sibling was asked to build (the per-statement funnel's proof). */ int metadataBuilds; boolean closed; @@ -96,6 +214,12 @@ public ConnectorScanPlanProvider getScanPlanProvider() { return scanPlanProvider; } + @Override + public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) { + calls.add("getScanPlanProvider"); + return scanPlanProvider; + } + @Override public boolean ownsHandle(ConnectorTableHandle handle) { return handle instanceof Handle; @@ -129,9 +253,20 @@ public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTa public Map getColumnHandles( ConnectorSession session, ConnectorTableHandle handle) { calls.add("getColumnHandles"); - Map handles = new LinkedHashMap<>(); - handles.put(COLUMN_NAME, new FlussColumnHandle(COLUMN_NAME, 0)); - return handles; + return lakeColumns; + } + + /** + * Threads the pin the way a real MVCC connector does: a NEW handle carrying the snapshot. The + * empty-properties latest-pin form is the only one fluss can send, so that is the only one + * answered here — a caller that invented connector-specific options would go unnoticed otherwise. + */ + @Override + public ConnectorTableHandle applySnapshot(ConnectorSession session, + ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { + calls.add("applySnapshot:" + snapshot.getSnapshotId() + ":" + snapshot.getProperties()); + Handle lakeHandle = (Handle) handle; + return new Handle(lakeHandle.dbName, lakeHandle.tableName, snapshot.getSnapshotId()); } @Override From 3efa3f296c75fe2146b1ea0dafaca7d4af199d97 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 13:46:06 +0800 Subject: [PATCH 19/35] [fix](paimon) Claim the table handles this connector produces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connector.ownsHandle defaults to false, and this connector never overrode it. That was invisible while paimon was only ever a front-door catalog: the predicate exists so a GATEWAY connector can embed another as a sibling and route a foreign handle back to whoever made it, since the sibling's concrete handle type cannot be named across the plugin classloader split. The fluss connector reads a lake table by delegating to this one, so it asks that question about every handle it gets back — and got "not mine" about handles paimon had just produced. Every guard on the gateway side then falls through, and the first cast throws a ClassCastException naming the GATEWAY's handle type and two class loaders, with nothing to suggest the missing piece is a method here. Same one-liner the iceberg and hudi siblings behind the hms gateway already carry. No unit test could have caught this: a hand-written test double implements ownsHandle precisely because it has to, so the double is more capable than the real connector. It took an end-to-end read to surface. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../connector/paimon/PaimonConnector.java | 17 ++++ .../paimon/PaimonConnectorOwnsHandleTest.java | 78 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 79f15eae474fd7..34990f28fc25f7 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.api.ConnectorPartitionInfo; import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorValidationContext; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; @@ -254,6 +255,22 @@ public ConnectorMetadata getMetadata(ConnectorSession session) { properties, context, schemaAtMemo, latestSnapshotCache, partitionViewCache); } + /** + * True for a handle this connector produced (a {@link PaimonTableHandle}). Tested against this connector's + * OWN in-loader type, so a gateway connector that embeds this one as a sibling can route a foreign paimon + * handle here without casting it across the plugin classloader split. Returns false for any other + * connector's handle, so the gateway keeps looking. + * + *

The default is {@code false}, which for a sibling means every one of the gateway's guards silently + * fails open and the first cast throws a ClassCastException instead — so this is required of any connector + * used as a sibling, not an optimization. Same implementation as the iceberg and hudi siblings behind the + * hms gateway. + */ + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof PaimonTableHandle; + } + @Override public void invalidateTable(String dbName, String tableName) { // REFRESH TABLE (and, via the generic PluginDrivenExternalCatalog DDL hook, a Doris-issued diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java new file mode 100644 index 00000000000000..a3f7955a8e49f3 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +/** + * Whether this connector claims the handles it produces, which is what lets a gateway connector + * embed it as a sibling: the gateway cannot name this module's handle type across the plugin classloader + * split, so it routes a handle by asking each sibling to test its own in-loader type. + * + *

Asserted rather than left to the SPI default, because the default is {@code false} and the failure it + * causes points away from here. A sibling that disowns its own handles makes every guard on the gateway + * side fail open, and the first cast throws a ClassCastException naming the GATEWAY's handle type and two + * class loaders — with nothing to suggest that the missing piece is a method this connector never + * overrode. That is not hypothetical: it is what happened the first time the fluss connector read a lake + * table through this one end to end. + */ +public class PaimonConnectorOwnsHandleTest { + + @Test + public void claimsItsOwnTableHandle() { + PaimonConnector connector = new PaimonConnector(Collections.emptyMap(), context()); + + Assertions.assertTrue(connector.ownsHandle(new PaimonTableHandle( + "db1", "t1", Collections.emptyList(), Collections.emptyList()))); + } + + @Test + public void disownsAnotherConnectorsHandle() { + // The gateway asks its siblings in turn, so answering yes to a foreign handle would route it to the + // wrong connector instead of leaving the gateway to keep looking. + PaimonConnector connector = new PaimonConnector(Collections.emptyMap(), context()); + + Assertions.assertFalse(connector.ownsHandle(new ForeignHandle())); + } + + /** Stands in for whatever another connector's handle happens to be; only its type matters here. */ + private static final class ForeignHandle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + } + + /** The connector wraps whatever context it is given, so it cannot be null; nothing here reads it. */ + private static ConnectorContext context() { + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "test_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }; + } +} From 927de84557425d6ef7a99d9f697e65be96365f40 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 13:46:19 +0800 Subject: [PATCH 20/35] [fix](be) Pick the table reader per scan range, not per scan node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FileScannerV2 built its table reader once, from the first range, and reused it for every range after that. One scan node can be given ranges of more than one table format: a fluss union read plans the table's lake half through the paimon connector and its log half itself, and both arrive as ranges of the same scan. Whichever range came first then decided the reader for all of them, and the other format's ranges were handed to a reader that does not understand them. That does not fail cleanly — it fails as whatever that reader makes of a foreign range. Here it was paimon's, reporting an unsupported file format for a fluss range that carries no paimon parameters at all. Which ranges share a scanner is up to the engine's assignment, so the same query succeeded or failed by how the ranges happened to be dealt out, and changing the projected columns could flip it either way. The reader now follows the range's table format. The expression contexts are deliberately not rebuilt: they are per-scanner and format-independent, and _init_expr_ctxes is not idempotent. Verified by disabling the rebuild and rerunning the suites: only the union read fails, with exactly the original error, and the four fluss suites that do not mix formats stay green. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- be/src/exec/scan/file_scanner_v2.cpp | 19 +++++++++++++++++++ be/src/exec/scan/file_scanner_v2.h | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 3866b42f9f7b76..0b0a08c604522e 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -405,6 +405,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { if (_first_scan_range) { RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader)); DORIS_CHECK(_table_reader != nullptr); + _table_reader_format = table_format_name(_current_range); RETURN_IF_ERROR(_init_expr_ctxes()); RETURN_IF_ERROR(_init_table_reader(_current_range)); } @@ -504,6 +505,24 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { DORIS_CHECK(_table_reader != nullptr); _current_range_path = _current_range.path; + // The reader is chosen by the range's table format, not the node's, because one node can be + // given both: a fluss union read plans its lake half through the paimon connector and its log + // half itself, and both land here as ranges of the same scan. Built once from the first range, + // the reader would then be handed a range of the other format -- which does not fail cleanly. + // It fails as whatever that reader makes of a foreign range, e.g. paimon's reporting an + // unsupported file format for a fluss range that has no paimon parameters at all. Which + // ranges share a scanner is up to the engine's assignment, so the same query succeeds or + // fails by how the ranges happened to be dealt out. + auto table_format = table_format_name(_current_range); + if (table_format != _table_reader_format) { + RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader)); + DORIS_CHECK(_table_reader != nullptr); + _table_reader_format = std::move(table_format); + // Same init the first reader got. The expression contexts are NOT rebuilt: they are + // per-scanner and format-independent, and _init_expr_ctxes is not idempotent. + RETURN_IF_ERROR(_init_table_reader(_current_range)); + } + const auto format_type = get_range_format_type(*_params, _current_range); _init_adaptive_batch_size_state(format_type); if (_block_size_predictor != nullptr) { diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 92edbc1a1817f9..f3a56542022771 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -181,6 +181,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + // The table format _table_reader was built for. A scan node may mix table formats -- a fluss + // union read gives one node its lake half as paimon ranges and its log half as fluss ones -- and + // the reader is format-specific, so it is rebuilt whenever this stops matching the range. + std::string _table_reader_format; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to From 03e73d6e1789e1ade5619b4d47298770e2c3cae6 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 13:46:34 +0800 Subject: [PATCH 21/35] [fix](fluss) Fail loudly when the lake sibling disowns its own handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every guard in the metadata gateway tells a lake handle from a fluss one by asking the sibling whether the handle is its own. Connector.ownsHandle defaults to false, so a sibling that never overrode it answers "not mine" about handles it has just produced: every guard falls through, and the first cast throws a ClassCastException naming two connectors and no cause. Checked once now, where the handle is born, in both places one is obtained — the $lake system-table handle and the union read's pinned lake handle. The message names the sibling's own class, so the fix is on the reader's screen instead of several layers away. The test double gains a switch for a sibling that inherits the default, because that is the shape of a real connector before it is first used as a sibling — and the only shape a double cannot have by accident. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussConnectorMetadata.java | 5 +++- .../fluss/FlussScanPlanProvider.java | 3 +++ .../doris/connector/fluss/LakeSibling.java | 26 +++++++++++++++++++ .../connector/fluss/FlussLakeTableTest.java | 24 +++++++++++++++++ .../connector/fluss/RecordingLakeSibling.java | 9 ++++++- 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 8881673b43a19b..6f3274679be2d1 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -225,7 +225,10 @@ public Optional getSysTableHandle(ConnectorSession session + flussHandle.getTableName() + "' does not exist yet: nothing has been tiered to the" + " lake. Start (or wait for) the fluss tiering service for this table"); } - return lakeHandle; + // From here on this handle travels back through the engine and returns to the guards below, which + // route it by asking the sibling whether it is its own. Checked once, here, where a failure still + // has a cause attached to it. + return Optional.of(LakeSibling.requireOwned(sibling, lakeHandle.get())); } @Override diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java index a77fef34b5601b..1e1a7fd5fc7427 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -303,6 +303,9 @@ private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableH + handle.getDatabaseName() + "." + handle.getTableName() + "' but its lake table" + " does not exist. The lake warehouse and the fluss cluster disagree; check the" + " table's table.datalake.* settings")); + // The lake half's ranges are planned by the sibling and mixed into this node's range list, where + // they are told apart from fluss's by which connector owns them. Checked at birth; see requireOwned. + LakeSibling.requireOwned(sibling, lakeHandle); // The pin is expressed in the SPI's own terms — a snapshot id and no connector options — so the // sibling translates it into whatever its SDK calls a snapshot. Nothing paimon-specific is named // here. The id needs no mapping either: what fluss records as the lake snapshot IS the id the lake diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java index 3c6553f7876dd6..03e333c10d69f1 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java @@ -20,6 +20,8 @@ import org.apache.doris.connector.api.Connector; import org.apache.doris.connector.api.ConnectorMetadata; import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; import java.util.function.Function; import java.util.function.Supplier; @@ -80,4 +82,28 @@ static T forward(ConnectorSession session, Connector sibling, return call(sibling, () -> call.apply(session.getStatementScope() .getOrCreateMetadata(key, () -> sibling.getMetadata(session)))); } + + /** + * Returns {@code handle} after checking the sibling recognizes it as its own, which is what every guard + * on this side relies on to tell a lake handle from a fluss one. + * + *

Checked where the handle is born rather than trusted, because {@link Connector#ownsHandle} defaults + * to {@code false}: a sibling that does not override it answers "not mine" about handles it just made, + * every guard silently fails open, and the first cast throws a ClassCastException naming two connectors + * and no cause. That is a real failure mode, not a hypothetical — the paimon connector had no override + * until it was first used as a sibling, and no unit test could see it (a hand-written stand-in + * implements the method precisely because it has to). + * + * @throws DorisConnectorException naming the sibling's own class, so the fix is on the reader's screen + */ + static T requireOwned(Connector sibling, T handle) { + if (!call(sibling, () -> sibling.ownsHandle(handle))) { + throw new DorisConnectorException("The lake connector " + + sibling.getClass().getName() + " does not recognize the table handle it just" + + " produced: it does not implement Connector.ownsHandle, which a connector must do" + + " to be usable as a sibling. Without it the fluss connector cannot tell its handles" + + " apart from its own"); + } + return handle; + } } diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java index 74fda23b49a79f..819efde568a3c7 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java @@ -208,6 +208,30 @@ public void anUntieredLakeTableFailsLoud() { Assertions.assertTrue(failure.getMessage().contains("db.lake_table"), failure.getMessage()); } + @Test + public void siblingThatDisownsItsHandleFailsLoud() { + // Connector.ownsHandle defaults to false, so a connector that never overrode it disowns the handles + // it just produced. Every guard on this side then silently fails open and the first cast throws a + // ClassCastException naming two class loaders and no cause -- which is exactly what the paimon + // connector did the first time it was used as a sibling for real. Caught where the handle is born, + // the message names the class that has to change. + FlussConnectorMetadata metadata = new FlussConnectorMetadata(withLakeTable(), + FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + sibling.claimsItsOwnHandles = false; + builtSiblings.add(sibling); + return sibling; + }, + handle -> null); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("ownsHandle"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains(RecordingLakeSibling.class.getName()), + failure.getMessage()); + } + @Test public void theSchemaOfALakeTableComesFromTheSibling() { FlussConnectorMetadata metadata = metadata(withLakeTable()); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java index 63f4e631d2b111..e4bacb68b9fdf3 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java @@ -220,9 +220,16 @@ public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle return scanPlanProvider; } + /** + * Set false to imitate a sibling that never overrode {@link Connector#ownsHandle} and so inherits the + * SPI default. That is what a real connector looks like before it is first used as a sibling, and it is + * invisible to a stand-in that always answers correctly. + */ + boolean claimsItsOwnHandles = true; + @Override public boolean ownsHandle(ConnectorTableHandle handle) { - return handle instanceof Handle; + return claimsItsOwnHandles && handle instanceof Handle; } @Override From b2a22df96c2a33dfd51c1c07ad87dafe06e4fc51 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 13:46:54 +0800 Subject: [PATCH 22/35] [test](fluss) Read a tiered table end to end, lake only and lake plus log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fluss regression cluster gains lakehouse storage: a paimon warehouse on a bind mount, the tiering service submitted as a Flink job while the fixtures are built, and five lake tables. Building them takes three steps, and both of the unobvious ones are load-bearing. The rows that belong in paimon are written first; the paimon row COUNTS are then polled until they match, rather than waiting for a snapshot to exist, because tiering commits what it has consumed so far and a fixture frozen half-tiered would leave a lake-only table with a log tail on some runs and not others. Only then is the tiering job cancelled, and only then is the log tail written — left running, the service would keep consuming the tail, and a suite asserting that a table reads as "lake plus log" would decay into one asserting "lake only" as the environment aged. test_fluss_lake_only reads the lake through the sibling: its rows, the three system columns fluss adds, and the type identity between a table and its $lake — required to hold, and until now checked only by reading both mappings side by side rather than against the paimon connector running. test_fluss_union_log leans on comparing the two read modes. required reads lake plus log; disabled replays the whole fluss log, which still holds everything because tiering copies rather than moves. Two entirely different readers over one table, so a seam that is off by a row in either direction makes them disagree — an assertion a single-mode suite cannot make, and one the hand-written sibling in the unit tests cannot make either. Four environment details that each cost a full restart to find are recorded where they were needed: paimon builds a hadoop Configuration even for a directory warehouse; the fluss image chowns /opt/fluss to uid 9999 but never declares USER, so it runs as root and its paimon directories lock out the flink containers; tiering a primary-key table reads kv snapshot FILES, so the flink cluster needs remote.data.dir mounted; and a warehouse path with no scheme is read as HDFS, which fails at scan time rather than at catalog creation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 50 +++- .../docker-compose/fluss/build-images.sh | 117 ++++++++-- .../docker-compose/fluss/flink/Dockerfile | 10 +- .../docker-compose/fluss/fluss.env.tpl | 29 ++- .../docker-compose/fluss/fluss.yaml.tpl | 51 +++++ .../fluss/scripts/run-init-sql.sh | 141 +++++++++++- .../fluss/sql/init-lake-tail.sql | 79 +++++++ .../docker-compose/fluss/sql/init.sql | 164 +++++++++++++- .../fluss/sql/lake-row-counts.sql | 52 +++++ .../thirdparties/run-thirdparties-docker.sh | 14 +- .../fluss/test_fluss_catalog.groovy | 7 +- .../fluss/test_fluss_lake_only.groovy | 192 ++++++++++++++++ .../fluss/test_fluss_union_log.groovy | 214 ++++++++++++++++++ 13 files changed, 1069 insertions(+), 51 deletions(-) create mode 100644 docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql create mode 100644 docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index 9169f0f6d4c603..f306d12cf1e975 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -21,9 +21,13 @@ under the License. Stack: ZooKeeper, a fluss coordinator server, one fluss tablet server, and a Flink cluster (jobmanager, taskmanager, sql-client). The sql-client container -runs `sql/init.sql` once and then idles; its healthcheck only turns green after +builds the fixtures once and then idles; its healthcheck only turns green after every statement succeeded, so `--wait` gates on the fixtures being complete. +The fluss cluster is lakehouse-enabled: `datalake.format: paimon` with a +filesystem warehouse under `data/paimon`, and the sql-client container runs the +fluss lake tiering service as a Flink job while building the fixtures. + ## Prerequisite: a built fluss checkout Fluss 1.0 is not released yet, so neither image can be pulled. `build-images.sh` @@ -31,9 +35,14 @@ builds both from a local source tree, which must be packaged first: ```bash git clone https://github.com/apache/fluss.git -mvn -f fluss/pom.xml -pl fluss-dist,fluss-flink/fluss-flink-1.20 -am package -DskipTests +mvn -f fluss/pom.xml \ + -pl fluss-dist,fluss-flink/fluss-flink-1.20,fluss-flink/fluss-flink-tiering,fluss-lake/fluss-lake-paimon \ + -am package -DskipTests ``` +The paimon runtime the tiering job needs (`paimon-flink-1.20`) is resolved from +the local maven repository, or downloaded from Maven Central when it is missing. + When fluss 1.0 ships, this step and `build-images.sh` are replaced by the official `apache/fluss` and Flink images. @@ -68,14 +77,18 @@ enableFlussTest=true The servers advertise `:`, because Doris FE/BE run on the host rather than inside the compose network. -`remote.data.dir` is bind mounted at the same absolute path inside the -containers and on the host (`data/remote`): Doris BE reads the kv snapshots and -remote log segments written there directly, so the two sides must agree on the -path string. +Two directories are bind mounted at the same absolute path inside the containers +and on the host, because Doris reads the files in them directly and the path +string is recorded rather than translated: + +| Directory | Written by | Read by | +|---|---|---| +| `data/remote` (`remote.data.dir`) | fluss servers | Doris BE — kv snapshots, remote log segments | +| `data/paimon` (`datalake.paimon.warehouse`) | the tiering job | Doris FE/BE through the paimon connector | ## Fixtures -`sql/init.sql` recreates database `fluss_test` from scratch on every start: +The fixtures recreate database `fluss_test` from scratch on every start: | Table | Shape | |---|---| @@ -86,8 +99,27 @@ path string. | `pk_basic` | primary-key table, one updated row and one deleted row | | `pk_types` | primary-key table with the same type coverage as `log_types` | | `pk_part` | primary-key table partitioned by `dt`, with an update and a delete inside a partition | - -Data-lake tables and the tiering service are added when union read lands. +| `lake_log` | lake table, 4 rows tiered + 2 in the log, 3 buckets (some bucket has no tail) | +| `lake_cold` | lake table read entirely from the lake — no log tail at all | +| `lake_types` | lake table with the full type coverage; non-NULL rows tiered, the all-NULL row in the log | +| `lake_part` | lake table partitioned by `dt`; only `20260101` has a log tail | +| `lake_pk` | primary-key lake table — union read is not implemented for it, so it pins the refusal | + +### Lake tables are frozen half in, half out + +Building them takes three steps (`scripts/run-init-sql.sh`): + +1. `sql/init.sql` writes the rows that belong in paimon, tiering service running; +2. `sql/lake-row-counts.sql` is polled until paimon holds every one of them; +3. the tiering job is cancelled, and only then does `sql/init-lake-tail.sql` + write the rows that must stay in the fluss log. + +Both the counting and the cancelling are load-bearing. Left running, the tiering +service would keep consuming the tail, and a suite asserting that a table is read +as "lake plus log" would quietly become one asserting "lake only" — passing or +failing by how long the environment had been up. And waiting for *a* paimon +snapshot rather than for the *row counts* would freeze some fixtures half-tiered, +which is the same flakiness one step earlier. ### Primary-key tables come with a kv snapshot diff --git a/docker/thirdparties/docker-compose/fluss/build-images.sh b/docker/thirdparties/docker-compose/fluss/build-images.sh index 6676248bd6fe13..dc82ac2064907e 100755 --- a/docker/thirdparties/docker-compose/fluss/build-images.sh +++ b/docker/thirdparties/docker-compose/fluss/build-images.sh @@ -29,22 +29,50 @@ # FLUSS_VERSION fluss version in that checkout (default 1.0-SNAPSHOT) # FLINK_BASE_IMAGE base Flink image (default flink:1.20.0-scala_2.12-java17) # FLUSS_FLINK_CONNECTOR_MODULE fluss connector module matching the base image -# FLUSS_DOCKER_REUSE_IMAGES 1 = skip the build when both tags already exist +# FLUSS_DOCKER_REUSE_IMAGES 1 = skip an image whose tag already exists +# (decided per image; delete a tag to rebuild just it) +# MAVEN_REPO local maven repository (default ~/.m2/repository) ################################################################ set -eo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -# Image tags live in fluss.env.tpl so the compose file and this script cannot -# drift apart. The template's other entries reference variables that are empty -# here; only the two image tags are read. +# Image tags and the paimon version live in fluss.env.tpl so the compose file and +# this script cannot drift apart. The template's other entries reference +# variables that are empty here; only those three literals are read. # shellcheck source=/dev/null . "${SCRIPT_DIR}/fluss.env.tpl" FLUSS_VERSION="${FLUSS_VERSION:-1.0-SNAPSHOT}" FLINK_BASE_IMAGE="${FLINK_BASE_IMAGE:-flink:1.20.0-scala_2.12-java17}" FLUSS_FLINK_CONNECTOR_MODULE="${FLUSS_FLINK_CONNECTOR_MODULE:-fluss-flink-1.20}" +MAVEN_REPO="${MAVEN_REPO:-${HOME}/.m2/repository}" +# Both fluss and paimon name their Flink artifacts after the Flink minor version, +# so deriving it from the fluss module keeps the paimon jar in step with the base +# image whenever the module is overridden. +FLINK_MINOR_VERSION="${FLUSS_FLINK_CONNECTOR_MODULE##*-}" + +# Resolves a maven artifact into a directory: the local repository first, so a +# machine that has already built Doris or fluss needs no network, then central. +resolve_maven_jar() { + local group_path="$1" artifact="$2" version="$3" dest_dir="$4" + local jar="${artifact}-${version}.jar" + local local_path="${MAVEN_REPO}/${group_path}/${artifact}/${version}/${jar}" + + mkdir -p "${dest_dir}" + if [[ -f "${local_path}" ]]; then + cp "${local_path}" "${dest_dir}/" + echo " ${jar} (from ${MAVEN_REPO})" + return 0 + fi + local url="https://repo1.maven.org/maven2/${group_path}/${artifact}/${version}/${jar}" + echo " ${jar} (downloading ${url})" + if ! curl -fsSL -o "${dest_dir}/${jar}" "${url}"; then + echo "ERROR: could not resolve ${jar} locally or from central" >&2 + return 1 + fi +} if [[ -z "${DOCKER_USE_SUDO+x}" ]]; then if [[ "$(uname -s)" == "Darwin" ]]; then @@ -66,9 +94,26 @@ image_exists() { docker_cli image inspect "$1" >/dev/null 2>&1 } -if [[ "${FLUSS_DOCKER_REUSE_IMAGES}" == "1" ]] && - image_exists "${FLUSS_SERVER_IMAGE}" && image_exists "${FLUSS_FLINK_IMAGE}"; then - echo "Reusing existing images ${FLUSS_SERVER_IMAGE} and ${FLUSS_FLINK_IMAGE}" +# Reuse is decided per image, because the two are built from different inputs: +# the server image from fluss-dist, the flink image from the connector, paimon +# and the tiering jar. Rebuilding one to pick up a change in the other wastes +# minutes, and -- since the base images have to be pulled -- fails outright on a +# machine that can reach the local checkout but not a registry. +should_build() { + local image="$1" + if [[ "${FLUSS_DOCKER_REUSE_IMAGES}" == "1" ]] && image_exists "${image}"; then + echo "Reusing existing image ${image}" + return 1 + fi + return 0 +} + +BUILD_SERVER=0 +BUILD_FLINK=0 +should_build "${FLUSS_SERVER_IMAGE}" && BUILD_SERVER=1 +should_build "${FLUSS_FLINK_IMAGE}" && BUILD_FLINK=1 + +if ((BUILD_SERVER == 0 && BUILD_FLINK == 0)); then exit 0 fi @@ -81,31 +126,61 @@ fi DIST_DIR="${FLUSS_SOURCE_DIR}/fluss-dist/target/fluss-${FLUSS_VERSION}-bin/fluss-${FLUSS_VERSION}" CONNECTOR_JAR="${FLUSS_SOURCE_DIR}/fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE}/target/${FLUSS_FLINK_CONNECTOR_MODULE}-${FLUSS_VERSION}.jar" - -if [[ ! -d "${DIST_DIR}" || ! -f "${CONNECTOR_JAR}" ]]; then +# The lake half of the environment: the tiering job that moves fluss data into +# paimon, and the fluss-side paimon writer it loads. +TIERING_JAR="${FLUSS_SOURCE_DIR}/fluss-flink/fluss-flink-tiering/target/fluss-flink-tiering-${FLUSS_VERSION}.jar" +LAKE_PAIMON_JAR="${FLUSS_SOURCE_DIR}/fluss-lake/fluss-lake-paimon/target/fluss-lake-paimon-${FLUSS_VERSION}.jar" + +MISSING=() +[[ -d "${DIST_DIR}" ]] || MISSING+=("${DIST_DIR}") +for jar in "${CONNECTOR_JAR}" "${TIERING_JAR}" "${LAKE_PAIMON_JAR}"; do + [[ -f "${jar}" ]] || MISSING+=("${jar}") +done + +if ((${#MISSING[@]} > 0)); then echo "ERROR: fluss build output is missing:" >&2 - [[ -d "${DIST_DIR}" ]] || echo " ${DIST_DIR}" >&2 - [[ -f "${CONNECTOR_JAR}" ]] || echo " ${CONNECTOR_JAR}" >&2 + printf ' %s\n' "${MISSING[@]}" >&2 echo " Build them with:" >&2 - echo " mvn -f ${FLUSS_SOURCE_DIR}/pom.xml -pl fluss-dist,fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE} -am package -DskipTests" >&2 + echo " mvn -f ${FLUSS_SOURCE_DIR}/pom.xml -pl fluss-dist,fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE},fluss-flink/fluss-flink-tiering,fluss-lake/fluss-lake-paimon -am package -DskipTests" >&2 exit 1 fi BUILD_CONTEXT="$(mktemp -d)" trap 'rm -rf "${BUILD_CONTEXT}"' EXIT -echo "Building ${FLUSS_SERVER_IMAGE} from ${DIST_DIR}" -mkdir -p "${BUILD_CONTEXT}/server" -cp -r "${DIST_DIR}" "${BUILD_CONTEXT}/server/build-target" -cp "${FLUSS_SOURCE_DIR}/docker/fluss/Dockerfile" "${BUILD_CONTEXT}/server/Dockerfile" -cp "${FLUSS_SOURCE_DIR}/docker/fluss/docker-entrypoint.sh" "${BUILD_CONTEXT}/server/docker-entrypoint.sh" -docker_cli build -t "${FLUSS_SERVER_IMAGE}" "${BUILD_CONTEXT}/server" +if ((BUILD_SERVER == 1)); then + echo "Building ${FLUSS_SERVER_IMAGE} from ${DIST_DIR}" + mkdir -p "${BUILD_CONTEXT}/server" + cp -r "${DIST_DIR}" "${BUILD_CONTEXT}/server/build-target" + cp "${FLUSS_SOURCE_DIR}/docker/fluss/Dockerfile" "${BUILD_CONTEXT}/server/Dockerfile" + cp "${FLUSS_SOURCE_DIR}/docker/fluss/docker-entrypoint.sh" "${BUILD_CONTEXT}/server/docker-entrypoint.sh" + docker_cli build -t "${FLUSS_SERVER_IMAGE}" "${BUILD_CONTEXT}/server" +fi + +if ((BUILD_FLINK == 0)); then + exit 0 +fi -echo "Building ${FLUSS_FLINK_IMAGE} from ${FLINK_BASE_IMAGE} + $(basename "${CONNECTOR_JAR}")" -mkdir -p "${BUILD_CONTEXT}/flink/lib" +echo "Building ${FLUSS_FLINK_IMAGE} from ${FLINK_BASE_IMAGE}" +mkdir -p "${BUILD_CONTEXT}/flink/lib" "${BUILD_CONTEXT}/flink/opt" cp "${CONNECTOR_JAR}" "${BUILD_CONTEXT}/flink/lib/" +echo " $(basename "${CONNECTOR_JAR}")" +# Paimon runtime for the tiering job. fluss-lake-paimon is only the fluss->paimon +# writer: it carries no paimon of its own, so paimon-flink (which bundles paimon +# core) has to sit next to it, and paimon in turn builds every CatalogContext +# around a hadoop Configuration -- a plain directory warehouse still needs hadoop +# present. Same three jars upstream's quickstart image activates for paimon, +# minus paimon-s3: this warehouse is a bind-mounted directory. +cp "${LAKE_PAIMON_JAR}" "${BUILD_CONTEXT}/flink/lib/" +echo " $(basename "${LAKE_PAIMON_JAR}")" +resolve_maven_jar "org/apache/paimon" "paimon-flink-${FLINK_MINOR_VERSION}" \ + "${FLUSS_PAIMON_VERSION}" "${BUILD_CONTEXT}/flink/lib" +resolve_maven_jar "io/trino/hadoop" "hadoop-apache" \ + "${FLUSS_HADOOP_APACHE_VERSION}" "${BUILD_CONTEXT}/flink/lib" +cp "${TIERING_JAR}" "${BUILD_CONTEXT}/flink/opt/" +echo " opt/$(basename "${TIERING_JAR}")" cp "${SCRIPT_DIR}/flink/Dockerfile" "${BUILD_CONTEXT}/flink/Dockerfile" docker_cli build --build-arg "FLINK_BASE_IMAGE=${FLINK_BASE_IMAGE}" \ -t "${FLUSS_FLINK_IMAGE}" "${BUILD_CONTEXT}/flink" -echo "Built ${FLUSS_SERVER_IMAGE} and ${FLUSS_FLINK_IMAGE}" +echo "Built ${FLUSS_FLINK_IMAGE}" diff --git a/docker/thirdparties/docker-compose/fluss/flink/Dockerfile b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile index 369eb822fdb42d..346024e9cc8011 100644 --- a/docker/thirdparties/docker-compose/fluss/flink/Dockerfile +++ b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile @@ -16,9 +16,15 @@ # # Flink image used to prepare the fluss regression data: stock Flink plus the -# fluss connector. build-images.sh assembles lib/ from a local fluss checkout, -# because fluss 1.0 is not released yet. +# fluss connector, the paimon runtime and the lake tiering job. build-images.sh +# assembles both directories, lib/ from a local fluss checkout (fluss 1.0 is not +# released yet) and paimon from the maven repository. ARG FLINK_BASE_IMAGE=flink:1.20.0-scala_2.12-java17 FROM ${FLINK_BASE_IMAGE} COPY lib/ /opt/flink/lib/ + +# The tiering job is submitted with `flink run`, so it is a job artifact rather +# than a cluster dependency: it belongs in opt/, not lib/. Upstream's quickstart +# image puts it in the same place. +COPY opt/ /opt/flink/opt/ diff --git a/docker/thirdparties/docker-compose/fluss/fluss.env.tpl b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl index 44b6acaf2b9463..8dc2c07f302b5d 100644 --- a/docker/thirdparties/docker-compose/fluss/fluss.env.tpl +++ b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl @@ -17,8 +17,8 @@ # under the License. # Rendered to fluss.env by run-thirdparties-docker.sh (envsubst). -# build-images.sh also sources this template directly for the image tags, so -# keep the two FLUSS_*_IMAGE lines free of variable references. +# build-images.sh also sources this template directly, for the image tags and +# the paimon version, so keep those lines free of variable references. DOCKER_FLUSS_ZOOKEEPER_EXTERNAL_PORT=22181 DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT=19123 @@ -39,3 +39,28 @@ FLUSS_HOST_IP=${IP_HOST} # those files directly (primary-key table reads), so the directory is bind # mounted at the SAME absolute path inside the containers and on the host. FLUSS_REMOTE_DATA_DIR=${FLUSS_COMPOSE_DIR}/data/remote + +# The paimon warehouse the tiering service writes the lake tables into. Doris +# reads it from the host through the paimon connector, so -- exactly like +# remote.data.dir above -- the same absolute path has to resolve on both sides: +# the warehouse location is recorded in fluss table properties and handed to +# paimon verbatim, with nothing left to rewrite a container path into a host one. +# +# Two spellings, and both are needed. The bind mount is a plain path, while the +# warehouse Doris is told about must carry the file:// scheme: a location with no +# scheme is read as HDFS (StorageRegistry.fromScheme defaults a blank scheme to +# HDFS), and every data-file path paimon recorded then fails to normalize with +# "Unsupported schema: null" at scan time -- long after catalog creation. +FLUSS_PAIMON_WAREHOUSE_DIR=${FLUSS_COMPOSE_DIR}/data/paimon +FLUSS_PAIMON_WAREHOUSE=file://${FLUSS_COMPOSE_DIR}/data/paimon + +# Paimon build the flink image carries, matched to the one fluss-lake-paimon was +# compiled against (fluss-dist ships paimon-bundle at this version) and to Doris's +# own paimon.version, so all three read the same table format. +FLUSS_PAIMON_VERSION=1.3.1 + +# Paimon builds its CatalogContext around a hadoop Configuration whatever the +# catalog is, so even a plain directory warehouse needs hadoop on the classpath; +# without it the tiering job dies with NoClassDefFoundError the first time it +# writes. Upstream's quickstart image carries the same repackaged jar. +FLUSS_HADOOP_APACHE_VERSION=3.3.5-1 diff --git a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl index e071113fd132c2..5c751e8f96d486 100644 --- a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl +++ b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl @@ -45,6 +45,12 @@ services: container_name: doris--fluss-coordinator hostname: doris--fluss-coordinator command: coordinatorServer + # The image chowns all of /opt/fluss to uid 9999 but never declares USER, so + # it runs as root unless told otherwise -- and then the paimon table + # directories it creates in the shared warehouse belong to root, while the + # tiering job writing into them is the flink image's uid 9999. Same uid on + # both sides, and the two can share the warehouse. + user: "9999:9999" depends_on: doris--fluss-zookeeper: condition: service_healthy @@ -62,8 +68,23 @@ services: remote.data.dir: ${FLUSS_REMOTE_DATA_DIR} default.bucket.number: 3 default.replication.factor: 1 + # Lakehouse storage. The coordinator creates the paimon table when a + # datalake-enabled fluss table is created, so it needs the warehouse + # too, not just the tiering job. The plugin jars are already in the + # image: fluss-dist ships plugins/paimon (fluss-lake-paimon + + # paimon-bundle + shaded hadoop). + # + # These three settings are also what makes the tables READABLE by Doris: + # the coordinator copies its datalake.paimon.* config into every lake + # table's properties under a table. prefix, and that copy is the only + # place the fluss connector learns where the warehouse is. + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: ${FLUSS_PAIMON_WAREHOUSE} volumes: - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} healthcheck: test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] interval: 5s @@ -77,6 +98,8 @@ services: container_name: doris--fluss-tablet-server hostname: doris--fluss-tablet-server command: tabletServer + # Same uid as the coordinator and the flink containers; see there. + user: "9999:9999" depends_on: doris--fluss-coordinator: condition: service_healthy @@ -107,8 +130,15 @@ services: # tablet whose log has not advanced since its last snapshot is skipped # (KvTabletSnapshotTarget), and the fixtures stop writing after init. kv.snapshot.interval: 10s + # Same lakehouse settings as the coordinator: a tablet server reads them + # to decide the key encoding and bucketing a datalake table uses. + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: ${FLUSS_PAIMON_WAREHOUSE} volumes: - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} healthcheck: test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] interval: 5s @@ -130,6 +160,15 @@ services: jobmanager.rpc.address: doris--fluss-jobmanager rest.address: doris--fluss-jobmanager rest.bind-address: 0.0.0.0 + # The tiering job runs on this cluster and writes the paimon warehouse; the + # jobmanager builds the job graph, which opens the lake catalog. + volumes: + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + # Tiering a primary-key table reads the kv snapshot FILES rather than the + # change log, straight out of remote.data.dir -- the same way Doris BE reads + # them. Without this mount a log table tiers and a primary-key one fails + # with FileNotFoundException for a file that plainly exists on the host. + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro healthcheck: test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8081/overview >/dev/null"] interval: 5s @@ -153,6 +192,11 @@ services: taskmanager.numberOfTaskSlots: 4 taskmanager.memory.process.size: 2048m taskmanager.memory.task.off-heap.size: 128m + # Writes and commits the paimon files the tiering job produces, and reads the + # kv snapshots it tiers a primary-key table from (see the jobmanager). + volumes: + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro # The taskmanager RPC port is ephemeral, so health means "registered with # the jobmanager": that is also exactly what submitting a job needs. healthcheck: @@ -184,6 +228,9 @@ services: # Read-only, and only so that init can wait for the kv snapshots to be # written before declaring the environment ready. - FLUSS_REMOTE_DATA_DIR=${FLUSS_REMOTE_DATA_DIR} + # This container also submits the tiering job and waits for it to commit, + # so it needs both the warehouse path and the paimon database naming. + - FLUSS_PAIMON_WAREHOUSE=${FLUSS_PAIMON_WAREHOUSE} - | FLINK_PROPERTIES= jobmanager.rpc.address: doris--fluss-jobmanager @@ -192,6 +239,10 @@ services: - ./sql:/opt/fluss-sql:ro - ./scripts:/opt/fluss-scripts:ro - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro + # Writable, not read-only like the one above: `flink run` builds the + # tiering job graph in this container, and building it opens the lake + # catalog, which creates the warehouse directory if it is not there yet. + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} healthcheck: test: ["CMD-SHELL", "test -f /tmp/fluss-init/SUCCESS"] interval: 5s diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh index e1d783143c6084..14156c0ffa7ada 100755 --- a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -21,21 +21,49 @@ # fixtures through the Flink SQL client, then idles so that the compose # healthcheck has something to gate on. The SUCCESS marker is written only # after every statement succeeded. +# +# The lake fixtures are built in three steps, because what makes them useful is +# WHERE each row ends up: +# 1. init.sql writes the rows that belong in paimon, with the tiering service +# running; +# 2. the tiering service is stopped once it has committed all of them; +# 3. init-lake-tail.sql writes the rows that must stay in the fluss log. +# Stopping the service is the point: it freezes the division between the two +# halves. Left running, it would keep consuming the tail, and a suite asserting +# that a table is read as "lake plus log" would slowly turn into one asserting +# "lake only" -- passing or failing by how long the environment had been up. ################################################################ set -eo pipefail MARKER_DIR=/tmp/fluss-init SQL_TEMPLATE=/opt/fluss-sql/init.sql +LAKE_TAIL_TEMPLATE=/opt/fluss-sql/init-lake-tail.sql +LAKE_COUNTS_TEMPLATE=/opt/fluss-sql/lake-row-counts.sql JOBMANAGER_PORT=8081 WAIT_SECONDS=180 SQL_TIMEOUT_SECONDS=900 ATTEMPTS=3 # Primary-key fixtures whose buckets must have been snapshotted before the # environment counts as ready. See wait_for_kv_snapshots. -SNAPSHOT_TABLES=(pk_basic pk_types pk_part) +SNAPSHOT_TABLES=(pk_basic pk_types pk_part lake_pk) SNAPSHOT_WAIT_SECONDS=120 +# What each lake fixture must hold in paimon before the tail is written -- the +# row counts init.sql writes, merged where the table has a primary key. Keep in +# step with init.sql; a mismatch here stalls the environment instead of +# corrupting a fixture, and the timeout prints both sides. +LAKE_EXPECTED_ROWS=( + "lake_log=4" + "lake_cold=3" + "lake_types=1" + "lake_part=3" + "lake_pk=3" +) +LAKE_TIERING_WAIT_SECONDS=300 +TIERING_JAR_GLOB='/opt/flink/opt/fluss-flink-tiering-*.jar' +FLINK_BIN=/opt/flink/bin/flink + rm -rf "${MARKER_DIR}" mkdir -p "${MARKER_DIR}" @@ -53,10 +81,84 @@ wait_for_jobmanager() { wait_for_jobmanager -# The bootstrap address is only known at compose time, and the Flink SQL client -# does not expand environment variables inside SQL files. +# The bootstrap address and the warehouse path are only known at compose time, +# and the Flink SQL client does not expand environment variables inside SQL +# files. sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ "${SQL_TEMPLATE}" >"${MARKER_DIR}/init.sql" +sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ + "${LAKE_TAIL_TEMPLATE}" >"${MARKER_DIR}/init-lake-tail.sql" +sed "s|__FLUSS_PAIMON_WAREHOUSE__|${FLUSS_PAIMON_WAREHOUSE}|g" \ + "${LAKE_COUNTS_TEMPLATE}" >"${MARKER_DIR}/lake-row-counts.sql" + +# Cancels every job on the cluster. This cluster runs nothing but the tiering +# service, and a retry must not leave the previous attempt's job consuming the +# database the next attempt is about to drop and recreate. +cancel_all_jobs() { + local ids + ids="$("${FLINK_BIN}" list -r 2>/dev/null | grep -oE '[0-9a-f]{32}' || true)" + local id + for id in ${ids}; do + echo "Cancelling flink job ${id}" + "${FLINK_BIN}" cancel "${id}" >/dev/null 2>&1 || true + done +} + +# Submits the fluss -> paimon tiering service. Detached, because it is a +# streaming job that has to keep running while init.sql writes. +start_tiering_job() { + local jar + # shellcheck disable=SC2086 + jar="$(ls ${TIERING_JAR_GLOB} 2>/dev/null | head -n 1)" + if [[ -z "${jar}" ]]; then + echo "ERROR: no tiering jar matching ${TIERING_JAR_GLOB}" >&2 + return 1 + fi + echo "Submitting tiering service from ${jar}" + # The lake settings repeat the ones the fluss servers carry. They have to: + # the servers use them to create the paimon table, this job uses them to + # write it, and neither reads the other's configuration. + "${FLINK_BIN}" run -d "${jar}" \ + --fluss.bootstrap.servers "${FLUSS_BOOTSTRAP_SERVERS}" \ + --datalake.format paimon \ + --datalake.paimon.metastore filesystem \ + --datalake.paimon.warehouse "${FLUSS_PAIMON_WAREHOUSE}" +} + +# Waits until paimon holds every row init.sql wrote to a lake table, by counting +# them in the warehouse itself (sql/lake-row-counts.sql) rather than by checking +# that some snapshot exists. Tiering commits what it has consumed so far, so a +# snapshot proves only that it started; freezing a fixture half-tiered would +# leave a table meant to be lake-only with a log tail on some runs and not +# others. +wait_for_lake_rows() { + local waited=0 + local log="${MARKER_DIR}/lake-row-counts.log" + local expected missing + while :; do + missing="" + if timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh \ + -f "${MARKER_DIR}/lake-row-counts.sql" >"${log}" 2>&1; then + for expected in "${LAKE_EXPECTED_ROWS[@]}"; do + grep -qF "LAKEROWS:${expected}" "${log}" || missing="${missing} ${expected}" + done + if [[ -z "${missing}" ]]; then + echo "Tiered to paimon:${LAKE_EXPECTED_ROWS[*]}" + return 0 + fi + else + missing=" (count query failed)" + fi + if ((waited >= LAKE_TIERING_WAIT_SECONDS)); then + echo "ERROR: tiering did not reach the expected row counts after ${LAKE_TIERING_WAIT_SECONDS}s:${missing}" >&2 + echo "ERROR: last count output follows" >&2 + cat "${log}" >&2 || true + return 1 + fi + sleep 10 + waited=$((waited + 10)) + done +} # Waits until every primary-key fixture has a kv snapshot on disk. # @@ -107,13 +209,14 @@ wait_for_kv_snapshots() { done } -run_attempt() { - local log="$1" +run_sql() { + local sql="$1" + local log="$2" local status=0 # Timeout, because a write that the servers keep rejecting is retried by the # fluss client practically forever: without it the container just hangs. - timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh -f "${MARKER_DIR}/init.sql" 2>&1 | tee "${log}" + timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh -f "${sql}" 2>&1 | tee "${log}" status="${PIPESTATUS[0]}" if ((status != 0)); then return "${status}" @@ -126,13 +229,35 @@ run_attempt() { return 0 } +run_attempt() { + local attempt="$1" + + # A previous attempt's tiering job would still be consuming the database + # init.sql is about to drop. + cancel_all_jobs + start_tiering_job || return 1 + + run_sql "${MARKER_DIR}/init.sql" "${MARKER_DIR}/init-attempt-${attempt}.log" || return 1 + + echo "Fluss init SQL finished; waiting for the tiering service to commit" + wait_for_lake_rows || return 1 + + # Everything meant for the lake is in the lake. Stop tiering BEFORE writing + # the tail, so the tail stays in the fluss log for good. + cancel_all_jobs + + run_sql "${MARKER_DIR}/init-lake-tail.sql" \ + "${MARKER_DIR}/init-lake-tail-attempt-${attempt}.log" || return 1 + return 0 +} + # init.sql drops and recreates its database up front, so a retry always starts # from the same state. Retries exist because the tablet server may still be # registering with the coordinator when the ports are already open. for ((attempt = 1; attempt <= ATTEMPTS; attempt++)); do echo "Running fluss init SQL (attempt ${attempt}/${ATTEMPTS})" - if run_attempt "${MARKER_DIR}/init-attempt-${attempt}.log"; then - echo "Fluss init SQL finished; waiting for kv snapshots" + if run_attempt "${attempt}"; then + echo "Fluss fixtures written; waiting for kv snapshots" if ! wait_for_kv_snapshots; then exit 1 fi diff --git a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql new file mode 100644 index 00000000000000..37d8f1ebc11241 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql @@ -0,0 +1,79 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- The second half of the lake fixtures: the rows that must stay in the fluss +-- log rather than reach paimon. scripts/run-init-sql.sh runs this only after +-- the tiering service has committed everything init.sql wrote AND has been +-- stopped, so these rows stay where they land and a union read has both halves +-- to stitch for as long as the environment lives. +-- +-- lake_cold gets nothing on purpose: it is the fixture for a table the lake +-- already holds in full. + +SET 'table.dml-sync' = 'true'; +SET 'parallelism.default' = '2'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = '__FLUSS_BOOTSTRAP_SERVERS__' +); + +USE CATALOG fluss_catalog; +USE fluss_test; + +-- Two rows over three buckets: at least one bucket keeps an empty tail. +INSERT INTO lake_log VALUES + (5, 'hot5', CAST(5.50 AS DECIMAL(10, 2))), + (6, 'hot6', CAST(6.60 AS DECIMAL(10, 2))); + +-- The all-NULL row, so the union read covers NULLs arriving from the log half +-- while the non-NULL values of every type come from the lake half. +INSERT INTO lake_types VALUES + ( + 2, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); + +-- Only 20260101 gets a tail; 20260102 stays entirely in the lake. +INSERT INTO lake_part VALUES + (4, 'lp1c', '20260101'); + +-- Updates row 3 and deletes row 1, so the fluss-only view of this table differs +-- from what paimon holds: the two are compared against each other. +INSERT INTO lake_pk VALUES + (3, 'lp3-hot'); + +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM lake_pk WHERE id = 1; +SET 'execution.runtime-mode' = 'streaming'; diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index 34f8a1f6d9babe..928c0ec4989f42 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -19,8 +19,12 @@ -- it, they never write. __FLUSS_BOOTSTRAP_SERVERS__ is substituted by -- scripts/run-init-sql.sh. -- --- Tables backed by the data lake are added together with the tiering service --- when union read lands; everything here is fluss-only. +-- This is the first of two scripts. Everything a lake table should hold in +-- PAIMON is written here; init-lake-tail.sql then writes the rows that must +-- stay in the fluss log. scripts/run-init-sql.sh stops the tiering service in +-- between, which is what freezes the split -- otherwise the tail would drift +-- into the lake at the next tiering round and every assertion about how the +-- two halves divide would decay into "it depends when you ran it". SET 'table.dml-sync' = 'true'; SET 'parallelism.default' = '2'; @@ -307,3 +311,159 @@ INSERT INTO pk_part VALUES SET 'execution.runtime-mode' = 'batch'; DELETE FROM pk_part WHERE id = 4 AND dt = '20260102'; SET 'execution.runtime-mode' = 'streaming'; + +-- =========================================================================== +-- Lake tables. 'table.datalake.enabled' makes the fluss coordinator create a +-- matching paimon table and lets the tiering service move data into it; the +-- lake settings themselves (warehouse, metastore) come from the cluster config +-- and are copied into each table's properties, which is where the Doris +-- connector reads them from. +-- +-- Freshness is the lag the tiering service is asked to keep. Three minutes by +-- default, which every environment start would then have to wait out. +-- =========================================================================== + +-- --------------------------------------------------------------------------- +-- lake_log: the ordinary union-read fixture. Three buckets and only two rows in +-- the tail, so at least one bucket is fully tiered and must contribute no log +-- range at all, while the others resume where the lake stops. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_log ( + id INT, + name STRING, + price DECIMAL(10, 2) +) COMMENT 'fluss log table tiered into paimon' +WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_log VALUES + (1, 'lake1', CAST(1.10 AS DECIMAL(10, 2))), + (2, 'lake2', CAST(2.20 AS DECIMAL(10, 2))), + (3, 'lake3', CAST(3.30 AS DECIMAL(10, 2))), + (4, 'lake4', CAST(4.40 AS DECIMAL(10, 2))); + +-- --------------------------------------------------------------------------- +-- lake_cold: written once and never again, so after the tiering service has +-- caught up the lake holds the whole table and planning must emit no log range +-- whatsoever. One bucket, so "no log range" is an exact number and not a range. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_cold ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_cold VALUES + (1, 'cold1'), + (2, 'cold2'), + (3, 'cold3'); + +-- --------------------------------------------------------------------------- +-- lake_types: the type-parity fixture. The connector's fluss->Doris mapping is +-- required to equal fluss->paimon->Doris, so that the table and its $lake +-- sibling present one schema and not two; this is the only place that identity +-- is checked against the real paimon connector instead of by reading both +-- mappings side by side. One bucket keeps the split between the two halves +-- exact: everything here is tiered, the all-NULL row stays in the log. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_types ( + id INT, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ); + +-- --------------------------------------------------------------------------- +-- lake_part: partitioned lake table. The tail goes into one partition only, so +-- the other one is served entirely from the lake -- the two halves have to be +-- stitched per (partition, bucket) and not per table. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_part ( + id INT, + name STRING, + dt STRING +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_part VALUES + (1, 'lp1a', '20260101'), + (2, 'lp1b', '20260101'), + (3, 'lp2a', '20260102'); + +-- --------------------------------------------------------------------------- +-- lake_pk: primary-key table tiered into paimon. Merging a lake with a change +-- log BY KEY is not implemented, so the connector refuses this table unless the +-- lake is switched off -- the fixture exists to pin that refusal, to check that +-- the fluss-only read still returns the whole table, and to let $lake read the +-- paimon side directly. Row 2 is updated before tiering, so the lake already +-- holds a merged view rather than a raw change log. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk VALUES + (1, 'lp1'), + (2, 'lp2'), + (3, 'lp3'); + +INSERT INTO lake_pk VALUES + (2, 'lp2-lake'); diff --git a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql new file mode 100644 index 00000000000000..f42f0fc198d5fe --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql @@ -0,0 +1,52 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Counts the rows the tiering service has committed to paimon, read straight +-- from the warehouse rather than through fluss. scripts/run-init-sql.sh runs +-- this in a loop and greps the markers until every table reports the count +-- init.sql wrote. +-- +-- Counting is what makes the environment deterministic. "A paimon snapshot +-- exists" would not: the service commits what it has consumed so far, so a +-- fixture could be frozen half-tiered, and then a table meant to be entirely in +-- the lake would still have a log tail -- for some runs and not others. +-- +-- __FLUSS_PAIMON_WAREHOUSE__ is substituted by scripts/run-init-sql.sh. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; +SET 'parallelism.default' = '1'; + +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '__FLUSS_PAIMON_WAREHOUSE__' +); + +USE CATALOG paimon_catalog; +USE fluss_test; + +-- One statement, so one Flink job per poll instead of one per table. The marker +-- is a single token with no spaces because it is matched with a literal grep. +SELECT CONCAT('LAKEROWS:lake_log=', CAST(COUNT(*) AS STRING)) AS marker FROM lake_log +UNION ALL +SELECT CONCAT('LAKEROWS:lake_cold=', CAST(COUNT(*) AS STRING)) FROM lake_cold +UNION ALL +SELECT CONCAT('LAKEROWS:lake_types=', CAST(COUNT(*) AS STRING)) FROM lake_types +UNION ALL +SELECT CONCAT('LAKEROWS:lake_part=', CAST(COUNT(*) AS STRING)) FROM lake_part +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk=', CAST(COUNT(*) AS STRING)) FROM lake_pk; diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index ad1fc501e2525b..9929dffb14cc95 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -1482,9 +1482,10 @@ start_mariadb() { start_fluss() { local fluss_dir="${ROOT}/docker-compose/fluss" - # The compose file bind mounts remote.data.dir at the same absolute path it - # uses inside the containers, so Doris BE (running on the host) can read the - # kv snapshots and remote log segments the servers write there. + # The compose file bind mounts remote.data.dir and the paimon warehouse at + # the same absolute paths it uses inside the containers, so Doris (running on + # the host) can read the kv snapshots and remote log segments the servers + # write, and the lake files the tiering service writes. export FLUSS_COMPOSE_DIR="${fluss_dir}" envsubst <"${fluss_dir}/fluss.env.tpl" >"${fluss_dir}/fluss.env" set -a @@ -1504,9 +1505,10 @@ start_fluss() { FLUSS_DOCKER_REUSE_IMAGES="${FLUSS_DOCKER_REUSE_IMAGES:-1}" \ bash "${fluss_dir}/build-images.sh" - reset_data_dirs "${FLUSS_REMOTE_DATA_DIR}" - # The fluss image runs as uid 9999, the host directory is created by root. - sudo chmod 777 "${FLUSS_REMOTE_DATA_DIR}" + reset_data_dirs "${FLUSS_REMOTE_DATA_DIR}" "${FLUSS_PAIMON_WAREHOUSE_DIR}" + # The fluss and flink images run as uid 9999, the host directories are + # created by root. + sudo chmod 777 "${FLUSS_REMOTE_DATA_DIR}" "${FLUSS_PAIMON_WAREHOUSE_DIR}" sudo chmod +x "${fluss_dir}/scripts/run-init-sql.sh" compose_up_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" -d --wait diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy index 2f19c9b64937fa..9d1f9490d4cfa7 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -52,9 +52,14 @@ suite("test_fluss_catalog", "p0,external") { def tableRows = sql """show tables""" def tables = tableRows.collect { it[0] } for (String expected : ["log_basic", "log_types", "log_part", "log_empty", "pk_basic", "pk_types", - "pk_part"]) { + "pk_part", "lake_log", "lake_cold", "lake_types", "lake_part", "lake_pk"]) { assertTrue(tables.contains(expected), "table ${expected} missing: ${tables}") } + // A lake table is listed once, under its own name. The $lake reader is a way + // of reading it, not a second table, and a catalog listing that showed both + // would double every lake table for anything walking the schema. + assertTrue(tables.every { !it.toString().contains("\$") }, + "system tables leaked into show tables: ${tables}") // --- schema mapping ---------------------------------------------------- // desc rows are [Field, Type, Null, Key, Default, Extra]. diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy new file mode 100644 index 00000000000000..59c4ca9108a230 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading the lake side of a fluss table on its own, through `tbl$lake`. +// +// This is the first place the delegation runs for real. The fluss plugin bundles +// no paimon at all: it asks the plugin manager for a paimon connector, hands it +// synthesized catalog properties, and routes the scan to the handle that +// connector returns. In unit tests the sibling is a stand-in on the same class +// loader, so what only a deployed run can show is that a real plugin is found, +// that a handle crossing the plugin boundary is routed rather than cast, and +// that paimon's own ServiceLoader lookups resolve under the class loader the +// call is pinned to. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// are frozen: the tiering service is stopped before the log tail is written, so +// exactly the rows below are in paimon and no others ever will be. +suite("test_fluss_lake_only", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_lake_only" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only, and fuzzy sessions + // randomize this variable. + sql """set enable_file_scanner_v2 = true""" + + def scalarOf = { String query -> sql(query)[0][0].toString() } + + // --- the lake holds what was tiered, and nothing that came after --------- + // lake_log got four rows before the tiering service was stopped and two + // after. Reading the lake alone must return the first four: a $lake that + // quietly fell back to the fluss read would return all six. + def lakeRows = sql """select id, name, price from lake_log\$lake order by id""" + assertEquals(4, lakeRows.size()) + assertEquals(["1", "lake1", "1.10"], lakeRows[0].collect { it.toString() }) + assertEquals(["2", "lake2", "2.20"], lakeRows[1].collect { it.toString() }) + assertEquals(["3", "lake3", "3.30"], lakeRows[2].collect { it.toString() }) + assertEquals(["4", "lake4", "4.40"], lakeRows[3].collect { it.toString() }) + + // --- the three columns fluss adds to every lake table -------------------- + // They belong to the lake table and not to the fluss one, which is the whole + // reason the two are exposed as separate tables rather than one. + def systemColumns = sql """ + select count(*) from lake_log\$lake + where __bucket >= 0 and __bucket < 3 + and __offset >= 0 + and __timestamp is not null + """ + assertEquals("4", systemColumns[0][0].toString()) + + def lakeSchema = sql """desc lake_log\$lake""" + def lakeColumnNames = lakeSchema.collect { it[0].toString() } + assertEquals(["id", "name", "price", "__bucket", "__offset", "__timestamp"], lakeColumnNames) + + // The fluss table itself has none of them. + def flussColumnNames = sql("""desc lake_log""").collect { it[0].toString() } + assertEquals(["id", "name", "price"], flussColumnNames) + + // --- type parity between the two doors ---------------------------------- + // The connector's fluss->Doris mapping has to equal fluss->paimon->Doris, or + // `tbl` and `tbl$lake` present two different schemas for one table. Until now + // that was checked by reading both mappings side by side; here the second one + // is the paimon connector actually running. + def typesOf = { String table -> + def result = [:] + sql("""desc ${table}""").each { row -> result.put(row[0].toString(), row[1].toString()) } + return result + } + def flussTypes = typesOf("lake_types") + def lakeTypes = typesOf("lake_types\$lake") + flussTypes.each { column, type -> + assertEquals(type, lakeTypes.get(column), + "column ${column} is ${type} on the fluss table but ${lakeTypes.get(column)} on its lake") + } + assertEquals(flussTypes.size() + 3, lakeTypes.size()) + + // Parity of the values, not just of the declared types: row 1 of lake_types + // is compared against itself read the other way round, through fluss rather + // than paimon. Two decoders, one row, one set of literals. + def lakeTypeRow = sql """ + select count(*) from lake_types\$lake where id = 1 + and f_boolean = true + and f_tinyint = 1 and f_smallint = 2 and f_int = 3 and f_bigint = 4 + and f_float = cast(1.5 as float) and f_double = 2.5 + and f_decimal = 123.4567 + and f_char = 'char1' and f_string = 'string1' + and hex(f_binary) = '010203' and hex(f_bytes) = '0A0B' + and f_date = '2026-01-01' + -- Compared as text, not as a timestamp: an equality on a microsecond TIMESTAMP is pushed + -- into paimon and matches nothing there, while a range predicate on the same column and + -- the value itself are both right. That is the paimon connector's own behaviour (a plain + -- paimon catalog over this warehouse does the same), so pinning it here would assert + -- someone else's bug. The value is what this suite is about, and casting keeps the + -- comparison in Doris. + and cast(f_timestamp as string) = '2026-01-01 01:02:03.456789' + and f_timestamp_ltz is not null + and array_size(f_array) = 3 and f_array[1] = 1 and f_array[3] = 3 + and f_map['k1'] = 1 and f_map['k2'] = 2 + and struct_element(f_row, 'r_int') = 1 + and struct_element(f_row, 'r_string') = 'nested1' + """ + assertEquals("1", lakeTypeRow[0][0].toString()) + + // The all-NULL row was written after tiering stopped, so it is not here. + assertEquals("1", scalarOf("""select count(*) from lake_types\$lake""")) + + // --- a table the lake holds in full -------------------------------------- + def coldRows = sql """select id, name from lake_cold\$lake order by id""" + assertEquals([["1", "cold1"], ["2", "cold2"], ["3", "cold3"]], + coldRows.collect { row -> row.collect { it.toString() } }) + + // --- partitioning survives the delegation -------------------------------- + // The lake table is partitioned by the same column, so the partition value + // has to come back with its own row and not with a neighbour's. + def partRows = sql """select id, name, dt from lake_part\$lake order by id""" + assertEquals(3, partRows.size()) + assertEquals(["1", "lp1a", "20260101"], partRows[0].collect { it.toString() }) + assertEquals(["2", "lp1b", "20260101"], partRows[1].collect { it.toString() }) + assertEquals(["3", "lp2a", "20260102"], partRows[2].collect { it.toString() }) + + // Pruning is the sibling's, not fluss's: the predicate is pushed to the + // paimon connector, which owns the plan for this table. + def prunedPart = sql """select id from lake_part\$lake where dt = '20260101' order by id""" + assertEquals(["1", "2"], prunedPart.collect { it[0].toString() }) + + // --- a primary-key table's lake is its merged state at the tiering point -- + // Row 2 was updated before tiering, so the lake holds the update, not both + // versions. Row 3's later update and row 1's delete came after and are absent, + // which is exactly how this differs from the fluss-only read of the same table. + def pkLakeRows = sql """select id, name from lake_pk\$lake order by id""" + assertEquals([["1", "lp1"], ["2", "lp2-lake"], ["3", "lp3"]], + pkLakeRows.collect { row -> row.collect { it.toString() } }) + + // --- projection and aggregation through the sibling ---------------------- + assertEquals("4", scalarOf("""select count(*) from lake_log\$lake""")) + assertEquals("11.00", scalarOf("""select sum(price) from lake_log\$lake""")) + def namesOnly = sql """select name from lake_log\$lake where id > 2 order by name""" + assertEquals(["lake3", "lake4"], namesOnly.collect { it[0].toString() }) + + // --- tables with no lake ------------------------------------------------- + // A table with no lake never offers the sub-table, so the name does not resolve + // and the engine answers before the connector is asked anything. That is the + // deliberate choice: advertising $lake on every fluss table would offer a + // sub-table whose only possible outcome is an error. The connector still + // re-checks when it IS asked -- discovery and resolution are two round trips, + // and the lake can be switched off in between -- but that guard is unreachable + // from here, which is why it is pinned in the unit tests instead. + test { + sql """select * from log_basic\$lake""" + exception "Unknown sys table" + } + + // $lake is a way to read a table, not a table of its own: it must not appear + // in the catalog listing, or every tool that walks the schema would show each + // lake table twice. + def tableNames = sql("""show tables""").collect { it[0].toString() } + assertTrue(tableNames.contains("lake_log"), "lake_log missing from ${tableNames}") + assertTrue(tableNames.every { !it.contains("\$") }, + "system tables leaked into show tables: ${tableNames}") + + sql """drop catalog if exists ${catalogName}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy new file mode 100644 index 00000000000000..b0a7240e1fb53e --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading a tiered log table as its lake plus the log written after it. +// +// The lake half is planned by the paimon connector on a snapshot the fluss +// coordinator pinned, the log half by fluss from the offsets that snapshot +// stopped at, and both halves end up as ranges of one scan node -- so BE builds +// a paimon reader and a fluss reader for the same query. +// +// The load-bearing assertion is the comparison between the two modes. `required` +// reads lake plus log; `disabled` replays the whole fluss log, which still holds +// everything because tiering copies rather than moves. Two entirely different +// readers over one table: if the seam between the halves is off by a row in +// either direction -- a row counted twice, a row skipped -- the two disagree. +// A single-mode suite could not tell a correct seam from a plausible one. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// are frozen: the tiering service is stopped before the log tail is written. +suite("test_fluss_union_log", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String unionCatalog = "test_fluss_union_log" + String flussOnlyCatalog = "test_fluss_union_log_off" + + // Two catalogs rather than one reconfigured between queries: the read mode is + // a catalog property, and having both live at once is what lets the same query + // be run down both paths and compared. + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def scalarOf = { String query -> sql(query)[0][0].toString() } + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + + // --- both halves are planned, and the plan says which is which ----------- + def logPlan = planOf("""select * from lake_log""") + assertTrue(logPlan.contains("flussScan: unionRead=yes"), "not a union read: ${logPlan}") + assertTrue(logPlan.contains("mode=required"), "unexpected mode: ${logPlan}") + assertTrue(countIn(logPlan, "lakeSplits") >= 1, "no lake splits: ${logPlan}") + // Four rows were tiered across three buckets and two written after, so at + // least one bucket has caught up with the lake and must contribute nothing. + // Which buckets the writer chose is its own business, hence a bound. + def logRanges = countIn(logPlan, "logRanges") + assertTrue(logRanges >= 1 && logRanges <= 2, + "lake_log planned ${logRanges} log ranges over 3 buckets: ${logPlan}") + assertEquals(0, countIn(logPlan, "pkRanges")) + + // --- the seam: same rows down both paths --------------------------------- + def compareModes = { String query, int expectedRows -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(expectedRows, union.size(), "union read returned ${union.size()} rows for: ${query}") + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + return union + } + + // Four rows out of the lake and two out of the log, in one result. + def logRows = compareModes("select id, name, price from lake_log order by id", 6) + assertEquals(["1", "lake1", "1.10"], logRows[0]) + assertEquals(["4", "lake4", "4.40"], logRows[3]) + assertEquals(["5", "hot5", "5.50"], logRows[4]) + assertEquals(["6", "hot6", "6.60"], logRows[5]) + + // Aggregates run over both halves; a half silently dropped shows up here even + // when the ordered row list is not materialized. + assertEquals("6", scalarOf("""select count(*) from lake_log""")) + assertEquals("23.10", scalarOf("""select sum(price) from lake_log""")) + + // --- a table the lake already holds in full ------------------------------ + // Nothing was written after tiering caught up, so every bucket's log range is + // empty and planning must emit none at all rather than ranges that read + // nothing. + def coldPlan = planOf("""select * from lake_cold""") + assertTrue(coldPlan.contains("unionRead=yes"), "not a union read: ${coldPlan}") + assertTrue(countIn(coldPlan, "lakeSplits") >= 1, "no lake splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "logRanges"), "cold table still has log ranges: ${coldPlan}") + compareModes("select id, name from lake_cold order by id", 3) + + // --- every type crosses the seam ----------------------------------------- + // Row 1 comes back through paimon, row 2 through the fluss log: two decoders + // feeding one result set, which is also what pins the mapping parity on real + // values rather than on declared types. + compareModes("select id from lake_types order by id", 2) + assertEquals("1", scalarOf(""" + select count(*) from lake_types where id = 1 + and f_boolean = true + and f_tinyint = 1 and f_smallint = 2 and f_int = 3 and f_bigint = 4 + and f_float = cast(1.5 as float) and f_double = 2.5 + and f_decimal = 123.4567 + and f_char = 'char1' and f_string = 'string1' + and hex(f_binary) = '010203' and hex(f_bytes) = '0A0B' + and f_date = '2026-01-01' + -- Cast rather than compared as a timestamp: this row comes from the lake half, and an + -- equality on a microsecond TIMESTAMP pushed into paimon matches nothing there. See the + -- same note in test_fluss_lake_only: it is the paimon connector's behaviour, reproducible + -- through a plain paimon catalog, and not something this suite should pin. + and cast(f_timestamp as string) = '2026-01-01 01:02:03.456789' + and f_timestamp_ltz is not null + and array_size(f_array) = 3 and f_array[1] = 1 and f_array[3] = 3 + and f_map['k1'] = 1 and f_map['k2'] = 2 + and struct_element(f_row, 'r_int') = 1 + and struct_element(f_row, 'r_string') = 'nested1' + """)) + // The all-NULL row arrived after tiering stopped, so it is the log half's. + // Checked column by column: a null map read one column off shifts everything + // after it, and a row count would not notice. + assertEquals("1", scalarOf(""" + select count(*) from lake_types where id = 2 + and f_boolean is null and f_tinyint is null and f_smallint is null + and f_int is null and f_bigint is null and f_float is null + and f_double is null and f_decimal is null and f_char is null + and f_string is null and f_binary is null and f_bytes is null + and f_date is null and f_timestamp is null and f_timestamp_ltz is null + and f_array is null and f_map is null and f_row is null + """)) + + // --- partitioning, where the two halves prune differently ---------------- + // The fluss half is given the partitions the engine pruned to; the paimon half + // ignores that list and prunes on the pushed-down predicate instead. Both have + // to land on the same partition, which is what these two assertions separate: + // one partition has a log tail, the other is served entirely from the lake. + def partRows = compareModes("select id, name, dt from lake_part order by id", 4) + assertEquals(["1", "lp1a", "20260101"], partRows[0]) + assertEquals(["3", "lp2a", "20260102"], partRows[2]) + assertEquals(["4", "lp1c", "20260101"], partRows[3]) + + def tieredPartPlan = planOf("""select * from lake_part where dt = '20260102'""") + assertTrue(tieredPartPlan.contains("unionRead=yes"), "not a union read: ${tieredPartPlan}") + assertEquals(0, countIn(tieredPartPlan, "logRanges"), + "a fully tiered partition still has log ranges: ${tieredPartPlan}") + assertEquals(["3"], rowsOf("""select id from lake_part where dt = '20260102'""").collect { it[0] }) + + def tailPartPlan = planOf("""select * from lake_part where dt = '20260101'""") + assertEquals(1, countIn(tailPartPlan, "logRanges"), + "the partition with a tail lost its log range: ${tailPartPlan}") + assertEquals(["1", "2", "4"], + rowsOf("""select id from lake_part where dt = '20260101' order by id""").collect { it[0] }) + + // --- what is not supported yet fails loudly ------------------------------ + // Merging a lake with a change log BY KEY is not implemented. The refusal has + // to name the primary key and say what reading with the lake switched off + // would and would not give, because that fallback returns a partial table for + // a primary-key table rather than the whole one. + test { + sql """select * from lake_pk""" + exception "primary-key" + } + + // With the lake switched off the same table reads as the fluss-only merged + // view: row 2's pre-tiering update, row 3's post-tiering one, row 1 deleted. + def pkFlussOnly = rowsOf( + """select id, name from ${flussOnlyCatalog}.fluss_test.lake_pk order by id""") + assertEquals([["2", "lp2-lake"], ["3", "lp3-hot"]], pkFlussOnly) + + // --- required does not mean "every table has a lake" --------------------- + // A table with no lake at all is not an error in required mode: there is + // nothing to fall back FROM. Only a lake table whose snapshot cannot be read + // is. + def plainPlan = planOf("""select * from log_basic""") + assertTrue(plainPlan.contains("flussScan: unionRead=no"), "unexpected union read: ${plainPlan}") + assertEquals(0, countIn(plainPlan, "lakeSplits")) + assertEquals("3", scalarOf("""select count(*) from log_basic""")) + + sql """switch internal""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} From 82aab4f2ee3abaac9156e9904879e78efc75bce1 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 16:01:13 +0800 Subject: [PATCH 23/35] [feat](fluss) Read a tiered primary-key table from fluss alone A primary-key table that is tiered into a lake was refused outright, which left a plain SELECT on it failing. Merging its lake with its change log BY KEY is still not implemented -- the two cannot simply be concatenated, since the log carries updates and deletes of rows the lake already holds -- but refusing is the wrong answer, because reading such a table from fluss alone returns the WHOLE table: fluss keeps a primary-key table's state in its own kv store, and tiering copies rows into the lake rather than moving them out. That is the opposite of a log table, where whatever tiering has aged out of the log lives only in the lake and a fluss-only read silently loses it. So auto and disabled now read it the way an untiered primary-key table is read, from the latest kv snapshot plus the log that followed. What is lost is speed, not rows. required still refuses it: that mode exists so a union-read test cannot pass without a union read. The decision is taken BEFORE the lake snapshot is asked for, because it cannot depend on the answer -- asked afterwards, a table whose tiering has not committed yet would be refused under required with "wait for the tiering service to commit", which is a dead end rather than a reason. The old refusal also told the reader that switching the lake off returns less than the whole table. That is true of a log table and false here. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussScanPlanProvider.java | 38 ++++-- .../connector/fluss/FlussSplitPlanTest.java | 112 +++++++++++++++--- 2 files changed, 124 insertions(+), 26 deletions(-) diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java index 1e1a7fd5fc7427..984420c454a7f9 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -78,8 +78,12 @@ * as fluss-only silently returns just the rows tiering has not moved yet, which looks like a working * query. {@code fluss.union_read.mode=disabled} is how a user asks for that fluss-only read on purpose. * - *

What is NOT here yet: the union of a PRIMARY-KEY table's lake with its log. That one cannot be a - * plain concatenation — the two halves have to be merged by key — so it is refused loudly. + *

A tiered PRIMARY-KEY table is read from fluss alone for now. Its two halves cannot be concatenated — + * the log carries updates and deletes of rows the lake already holds, so they have to be merged BY KEY — + * and that merge is not implemented. Falling back is safe here in a way it is not for a log table: fluss + * keeps a primary-key table's state in full, so the fluss-only read is the whole table, only slower than + * reading the lake's columnar files would be. {@code required} refuses such a table anyway, because that + * mode exists to make "did this actually read the lake?" answerable in a test. */ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { @@ -266,6 +270,29 @@ private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableH // Not a lake table, or the user asked for the fluss-only read explicitly. return null; } + if (handle.hasPrimaryKey()) { + // A primary-key table's lake is NOT read, and the fluss-only read that replaces it is the whole + // table rather than a part of it: fluss keeps the table's state in its own kv store, and tiering + // copies rows into the lake without taking them out of it, so the latest kv snapshot plus the + // change log after it is every row. That is the opposite of a log table, where whatever tiering + // has aged out of the log exists only in the lake and a fluss-only read silently loses it — which + // is why that case falls back only when the lake holds nothing yet. What is lost here is speed, + // not rows: the bucket's whole kv snapshot is fetched and merged instead of reading the lake's + // columnar files. Merging the two by key is a separate piece of work. + // + // Asked before the lake snapshot is, deliberately: the answer cannot depend on it. A table whose + // tiering has not committed yet would otherwise report the wrong reason under 'required', and + // waiting for that commit would not help. + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' is a primary-key table tiered into a lake, and '" + + FlussConnectorProperties.UNION_READ_MODE + "=required' asks for its lake and its" + + " change log to be read as one. Merging them by key is not implemented yet. Reading" + + " it from fluss alone still returns the whole table, so set the property to auto or" + + " disabled."); + } + return null; + } LakeSnapshot snapshot; try { snapshot = adminOps.getReadableLakeSnapshot(handle.toTablePath()); @@ -280,13 +307,6 @@ private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableH // Nothing is in the lake, so the log holds everything: the fluss-only read is the whole table. return null; } - if (handle.hasPrimaryKey()) { - throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." - + handle.getTableName() + "' is a primary-key table tiered into a lake. Reading it" - + " requires merging the lake with the change log by key, which is not supported yet." - + " Set '" + FlussConnectorProperties.UNION_READ_MODE + "=disabled' to read only what" - + " the fluss log still holds, which is NOT the whole table."); - } String lakeFormat = handle.getDataLakeFormat(); if (lakeFormat == null || !PAIMON_LAKE_FORMAT.equalsIgnoreCase(lakeFormat)) { throw new DorisConnectorException("Cannot read table '" + handle.getDatabaseName() + "." diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java index 9fc29f5d760f22..b766dc0d40c473 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -349,30 +349,82 @@ public void prunedPartitionsOfPrimaryKeyTablesAreNotEvenAskedAbout() { adminOps.calls.toString()); } - // ---------------------------------------------------------------- what is refused, and why + // ------------------------------------------- tiered primary-key tables: read from fluss alone /** - * A tiered primary-key table is refused for the same reason a tiered log table is: the fluss-only - * read returns whatever has not been tiered away yet, which is a successful query with missing rows. - * The primary-key read being implemented does not change that. + * Merging a primary-key table's lake with its change log is not implemented, and what replaces it is + * NOT the "whatever has not been tiered away" fallback a log table would get: fluss keeps such a table's + * state in full, so its kv snapshot plus the log after it is every row. The lake is not consulted at all + * — a plan that quietly asked it would produce these same ranges, so the calls are asserted too, and the + * sibling factory fails loud on its own if planning ever tries to build one. */ @Test - public void tieredPrimaryKeyTableIsRefusedUntilTheUnionReadExists() { - adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) - .column("id", DataTypes.INT().copy(false)) - .column("v", DataTypes.STRING()) - .primaryKey("id") - .buckets(2, "id") - .property("table.datalake.enabled", "true") - .property("table.datalake.format", "paimon") - .property("table.datalake.paimon.metastore", "filesystem") - .property("table.datalake.paimon.warehouse", "/lake/warehouse") - .build()); + public void tieredPrimaryKeyTableIsReadFromFlussAlone() { + registerTieredPkTable(2); adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 12L, 25L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 12L); + assertPkRange(ranges.get(1), 1, 5L, 20L, 25L); + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + /** + * The fallback is the very read {@code disabled} asks for outright, down to the ranges — not a third + * code path that happens to look similar. + */ + @Test + public void theFallbackIsTheReadDisabledModeAsksForOutright() { + registerTieredPkTable(2); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 12L, 25L); + + List> auto = rangeProperties(plan(PK_TABLE, catalog())); + List> disabled = rangeProperties( + plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + Assertions.assertEquals(auto, disabled); + } + + /** + * {@code required} is what a regression test sets so that it cannot pass without the lake having been + * read. No primary-key table can satisfy it today, so it fails loud instead of falling back — and it does + * so whether or not tiering has committed anything, because waiting for that commit would not help. + */ + @Test + public void requiredModeRefusesATieredPrimaryKeyTable() { + registerTieredPkTable(2); DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, - () -> plan(PK_TABLE, catalog())); - Assertions.assertTrue(e.getMessage().contains("not supported yet"), e.getMessage()); + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("not implemented yet"), e.getMessage()); + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + /** + * The plan's own account of not having read the lake. Nothing else in the plan of a tiered primary-key + * table distinguishes it from one that was read as a union, which is what the follow-up work will do. + */ + @Test + public void explainShowsATieredPrimaryKeyTableWasReadWithoutItsLake() { + registerTieredPkTable(1); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 12L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals( + "flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=1, mode=auto\n", + output.toString()); } @@ -808,6 +860,23 @@ private void registerPkTable(TablePath tablePath, int buckets) { .build()); } + /** + * A primary-key table tiered into a lake. Deliberately does NOT mark a sibling as expected: its lake is + * never read, so reaching the sibling factory at all is a failure. + */ + private void registerTieredPkTable(int buckets) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(buckets, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + } + /** A primary-key table partitioned by {@code dt}, with partition ids 100, 101, ... in order. */ private void registerPartitionedPkTable(int buckets, String... partitionValues) { adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) @@ -869,6 +938,15 @@ private static void assertPkRange(ConnectorScanRange range, int bucket, long sna Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); } + /** The ranges' payloads, in order — what two plans have to agree on to be the same read. */ + private static List> rangeProperties(List ranges) { + List> properties = new ArrayList<>(ranges.size()); + for (ConnectorScanRange range : ranges) { + properties.add(range.getProperties()); + } + return properties; + } + /** Position of the first recorded call starting with {@code prefix}, or -1. */ private int indexOfCall(String prefix) { for (int i = 0; i < adminOps.calls.size(); i++) { From cbbab7d119a186280efc43fb516cceebf9d5498a Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 16:01:44 +0800 Subject: [PATCH 24/35] [test](fluss) Pin what a tiered primary-key table reads as The fixture's primary-key lake table now disagrees with its lake in all three ways a change log can: one tiered row is updated, another is deleted, and a key that the lake never saw is added. The last one is new here, and it is the case a merge of the two halves is most likely to get wrong -- the rows that exist only in the log have to be emitted exactly once no matter how many lake files the bucket has. The suite records what that table reads as today, next to what its lake holds, so the two can be seen not to be the same thing. Reading it from fluss alone is complete, so this baseline is also the answer a future lake+log merge has to reproduce row for row: when that lands, these recorded results must not change. Nothing in the result distinguishes a fluss-only read from a correct merge -- which is what makes it a baseline -- so the plan is asserted separately for which path actually ran. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 2 +- .../fluss/sql/init-lake-tail.sql | 10 +- .../docker-compose/fluss/sql/init.sql | 11 +- .../fluss/test_fluss_lake_pk.out | 27 ++++ .../fluss/test_fluss_lake_pk.groovy | 139 ++++++++++++++++++ 5 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index f306d12cf1e975..5a5f9cc4745e49 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -103,7 +103,7 @@ The fixtures recreate database `fluss_test` from scratch on every start: | `lake_cold` | lake table read entirely from the lake — no log tail at all | | `lake_types` | lake table with the full type coverage; non-NULL rows tiered, the all-NULL row in the log | | `lake_part` | lake table partitioned by `dt`; only `20260101` has a log tail | -| `lake_pk` | primary-key lake table — union read is not implemented for it, so it pins the refusal | +| `lake_pk` | primary-key lake table; its tail updates one tiered row, deletes another and adds a key the lake never saw | ### Lake tables are frozen half in, half out diff --git a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql index 37d8f1ebc11241..393bb1f325a0fe 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql @@ -69,10 +69,14 @@ INSERT INTO lake_types VALUES INSERT INTO lake_part VALUES (4, 'lp1c', '20260101'); --- Updates row 3 and deletes row 1, so the fluss-only view of this table differs --- from what paimon holds: the two are compared against each other. +-- The tail carries one of each way a change log can disagree with the lake it +-- follows: row 3 is updated, row 1 is deleted, and row 4 exists only here. All +-- three are what a merge of the two halves has to get right, and the fluss-only +-- read of this table -- which is the whole table, because fluss keeps a +-- primary-key table's state in full -- is the answer that merge must reproduce. INSERT INTO lake_pk VALUES - (3, 'lp3-hot'); + (3, 'lp3-hot'), + (4, 'lp4-hot'); SET 'execution.runtime-mode' = 'batch'; DELETE FROM lake_pk WHERE id = 1; diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index 928c0ec4989f42..c32128b0e5a07c 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -444,11 +444,12 @@ INSERT INTO lake_part VALUES -- --------------------------------------------------------------------------- -- lake_pk: primary-key table tiered into paimon. Merging a lake with a change --- log BY KEY is not implemented, so the connector refuses this table unless the --- lake is switched off -- the fixture exists to pin that refusal, to check that --- the fluss-only read still returns the whole table, and to let $lake read the --- paimon side directly. Row 2 is updated before tiering, so the lake already --- holds a merged view rather than a raw change log. +-- log BY KEY is not implemented, so this table is read from fluss alone -- which +-- is the WHOLE table rather than a part of it, because fluss keeps a primary-key +-- table's state in full. That read is the baseline the future merge has to +-- reproduce: $lake shows what paimon holds, the front door shows the answer. +-- Row 2 is updated before tiering, so the lake already holds a merged view +-- rather than a raw change log. -- --------------------------------------------------------------------------- CREATE TABLE lake_pk ( id INT NOT NULL, diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out new file mode 100644 index 00000000000000..9551d34afc79d0 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lake_side -- +1 lp1 +2 lp2-lake +3 lp3 + +-- !front_door -- +2 lp2-lake +3 lp3-hot +4 lp4-hot + +-- !fluss_only -- +2 lp2-lake +3 lp3-hot +4 lp4-hot + +-- !count -- +3 + +-- !projection -- +lp3-hot + +-- !deleted_row -- + +-- !log_only_key -- +4 lp4-hot + diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy new file mode 100644 index 00000000000000..1505b3ed76ce0f --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading a PRIMARY-KEY table that is tiered into a lake. +// +// Its two halves cannot be concatenated the way a log table's are: the log +// carries updates and deletes of rows the lake already holds, so reading both +// and adding them up returns superseded and deleted rows. Merging them BY KEY is +// not implemented, so such a table is read from fluss alone -- and that read is +// the WHOLE table, not a part of it, because fluss keeps a primary-key table's +// state in its own kv store and tiering copies rows into the lake rather than +// moving them out. (A log table has no such guarantee, which is why IT is read +// as a union and refused when the lake cannot be reached.) +// +// This suite is therefore two things at once. It checks today's read, and it +// pins the answer a future lake+log merge will have to reproduce row for row: +// the same queries, over a fixture whose lake and log deliberately disagree in +// every way they can. When that merge lands, these recorded results must not +// change -- that is what recording them is for. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// init-lake-tail.sql, and are frozen -- the tiering service is stopped before +// the tail is written, so the two halves stay apart for as long as the +// environment lives. +suite("test_fluss_lake_pk", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String autoCatalog = "test_fluss_lake_pk" + String flussOnlyCatalog = "test_fluss_lake_pk_off" + String requiredCatalog = "test_fluss_lake_pk_required" + + // Three catalogs, one per read mode, all live at once: the mode is a catalog + // property, and comparing what the modes return is most of the point here. + sql """drop catalog if exists ${autoCatalog}""" + sql """ + create catalog ${autoCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """drop catalog if exists ${requiredCatalog}""" + sql """ + create catalog ${requiredCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + + sql """switch ${autoCatalog}""" + sql """use fluss_test""" + // The C++ glue exists only for the v2 file scanner, and the session variable + // that picks between them is randomised by the fuzzy mode this pipeline runs. + sql """set enable_file_scanner_v2 = true""" + + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + + // --- the two halves, and what the table IS ------------------------------- + // The lake was frozen mid-history: it holds row 2 already updated (that + // update happened before tiering) and rows 1 and 3 as first written. + order_qt_lake_side """select id, name from lake_pk\$lake""" + + // The log tail that follows disagrees with the lake in all three possible + // ways: row 3 updated, row 1 deleted, row 4 added. The table is therefore + // neither half, and no concatenation of the two produces it -- adding them up + // would return the deleted row 1, two versions of row 3, and row 4. + order_qt_front_door """select id, name from lake_pk""" + + // --- auto falls back to exactly what disabled asks for ------------------- + // The same rows out of the same code path, not "something equivalent": this + // result and the one above have to stay identical to each other, and a + // fallback that quietly read something else would break one of them. + order_qt_fluss_only """select id, name from ${flussOnlyCatalog}.fluss_test.lake_pk""" + + // --- the plan says the lake was not read --------------------------------- + // Nothing in the RESULT distinguishes this read from a correct merge -- that + // is exactly what makes it a baseline -- so only the plan can say which one + // ran. One bucket, one primary-key range, no lake splits. + def plan = planOf("""select * from lake_pk""") + assertTrue( + plan.contains("flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=1, mode=auto"), + "not planned as a fluss-only primary-key read of one bucket: ${plan}") + + // --- required refuses rather than falling back --------------------------- + // That mode exists so a union-read test cannot pass without a union read. No + // primary-key table can satisfy it yet, and the refusal has to say so -- + // "wait for tiering to commit" would send the reader down a dead end. + test { + sql """select * from ${requiredCatalog}.fluss_test.lake_pk""" + exception "not implemented yet" + } + + // --- the ordinary things still work on this path ------------------------- + // Projection, predicates and count all go through the kv snapshot merged with + // the change log, which is a different reader from the log tables' and is + // exercised here for a lake table. + order_qt_count """select count(*) from lake_pk""" + order_qt_projection """select name from lake_pk where id = 3""" + // The deleted row stays gone under a predicate too, not only in a full scan: + // the delete is a log record, and applying it on one path but not the other is + // how a "row that only shows up sometimes" bug looks. + order_qt_deleted_row """select name from lake_pk where id = 1""" + order_qt_log_only_key """select id, name from lake_pk where id > 3""" + + sql """switch internal""" + sql """drop catalog if exists ${autoCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """drop catalog if exists ${requiredCatalog}""" +} From e754eb5e0c8bbcd382a71b8b481298326624c087 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 16:02:03 +0800 Subject: [PATCH 25/35] [test](fluss) Record the fluss suites' results in baseline files These suites asserted every result inline. That was a choice made before there was a cluster to run them on: a .out cannot be generated without one, and a hand-written .out is worse than none. There is a cluster now, so the results are recorded the way the rest of the regression tests record them. What is NOT a result stays in the code: the EXPLAIN anchors saying how a scan was planned, the refusal messages, the range-count bounds (which bucket a row lands in is fluss's choice, so they cannot be exact), and the two invariants that are comparisons rather than expectations -- that lake+log and fluss-only return the same rows, and that a table and its $lake report the same column types. Three values are deliberately left unrecorded because they are not reproducible: __bucket and __offset, which depend on the writer's bucket choice, and __timestamp, which is a wall clock. What is recorded of them is that every row has all three, within the range each must be in. The session time zone is now pinned, because TIMESTAMP_LTZ renders through it and the baseline holds what it rendered as. Recording the types tables whole also pins how a map, a struct and a decimal print, which the predicate-based assertions deliberately avoided. That is the trade: a rendering change now lands here and has to be re-recorded on purpose. In exchange the type coverage got stronger -- desc now pins the mapped types, and the same row read through fluss, through paimon and through a kv snapshot produces three baseline lines that are identical character for character. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/test_fluss_catalog.out | 71 ++++++++ .../fluss/test_fluss_lake_only.out | 117 ++++++++++++++ .../fluss/test_fluss_log_table.out | 54 +++++++ .../fluss/test_fluss_pk_table.out | 50 ++++++ .../fluss/test_fluss_union_log.out | 41 +++++ .../fluss/test_fluss_catalog.groovy | 62 +++---- .../fluss/test_fluss_lake_only.groovy | 115 ++++++------- .../fluss/test_fluss_log_table.groovy | 152 +++++------------- .../fluss/test_fluss_pk_table.groovy | 140 ++++++---------- .../fluss/test_fluss_union_log.groovy | 114 +++++-------- 10 files changed, 537 insertions(+), 379 deletions(-) create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_catalog.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_log_table.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_union_log.out diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out new file mode 100644 index 00000000000000..a3dfae055cc802 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out @@ -0,0 +1,71 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !databases -- +fluss +fluss_test +information_schema +mysql + +-- !tables -- +lake_cold +lake_log +lake_part +lake_pk +lake_types +log_basic +log_empty +log_part +log_types +pk_basic +pk_part +pk_types + +-- !desc_log_basic -- +id int Yes true \N +name text Yes true \N +price decimal(10,2) Yes true \N + +-- !desc_log_types -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes true \N +f_int int Yes true \N +f_bigint bigint Yes true \N +f_float float Yes true \N +f_double double Yes true \N +f_decimal decimal(20,4) Yes true \N +f_char char(5) Yes true \N +f_string text Yes true \N +f_binary text Yes true \N +f_bytes text Yes true \N +f_date date Yes true \N +f_timestamp datetime(6) Yes true \N +f_timestamp_ltz datetime(3) Yes true \N WITH_TIMEZONE +f_array array Yes true \N +f_map map Yes true \N +f_row struct Yes true \N + +-- !desc_log_part -- +id int Yes true \N +name text Yes true \N +dt text Yes true \N + +-- !desc_pk_basic -- +id int Yes true \N +name text Yes true \N +score double Yes true \N + +-- !tables_after_refresh -- +lake_cold +lake_log +lake_part +lake_pk +lake_types +log_basic +log_empty +log_part +log_types +pk_basic +pk_part +pk_types + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out new file mode 100644 index 00000000000000..0a56be161de721 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out @@ -0,0 +1,117 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lake_rows -- +1 lake1 1.10 +2 lake2 2.20 +3 lake3 3.30 +4 lake4 4.40 + +-- !system_columns -- +4 + +-- !desc_lake_log_lake -- +id int Yes true \N NONE +name text Yes true \N NONE +price decimal(10,2) Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !desc_lake_log -- +id int Yes true \N +name text Yes true \N +price decimal(10,2) Yes true \N + +-- !desc_lake_types -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes true \N +f_int int Yes true \N +f_bigint bigint Yes true \N +f_float float Yes true \N +f_double double Yes true \N +f_decimal decimal(20,4) Yes true \N +f_char char(5) Yes true \N +f_string text Yes true \N +f_binary text Yes true \N +f_bytes text Yes true \N +f_date date Yes true \N +f_timestamp datetime(6) Yes true \N +f_timestamp_ltz datetime(3) Yes true \N WITH_TIMEZONE +f_array array Yes true \N +f_map map Yes true \N +f_row struct Yes true \N + +-- !desc_lake_types_lake -- +id int Yes true \N NONE +f_boolean boolean Yes true \N NONE +f_tinyint tinyint Yes true \N NONE +f_smallint smallint Yes true \N NONE +f_int int Yes true \N NONE +f_bigint bigint Yes true \N NONE +f_float float Yes true \N NONE +f_double double Yes true \N NONE +f_decimal decimal(20,4) Yes true \N NONE +f_char char(5) Yes true \N NONE +f_string text Yes true \N NONE +f_binary text Yes true \N NONE +f_bytes text Yes true \N NONE +f_date date Yes true \N NONE +f_timestamp datetime(6) Yes true \N NONE +f_timestamp_ltz datetime(3) Yes true \N NONE +f_array array Yes true \N NONE +f_map map Yes true \N NONE +f_row struct Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !lake_types_row -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} + +-- !lake_types_count -- +1 + +-- !cold_rows -- +1 cold1 +2 cold2 +3 cold3 + +-- !part_rows -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 + +-- !part_pruned -- +1 +2 + +-- !pk_lake_rows -- +1 lp1 +2 lp2-lake +3 lp3 + +-- !lake_count -- +4 + +-- !lake_sum -- +11.00 + +-- !lake_names -- +lake3 +lake4 + +-- !tables -- +lake_cold +lake_log +lake_part +lake_pk +lake_types +log_basic +log_empty +log_part +log_types +pk_basic +pk_part +pk_types + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out b/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out new file mode 100644 index 00000000000000..eb56b8a8f21e60 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !basic_all -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !basic_count -- +3 + +-- !basic_names -- +alice +bob +carol + +-- !basic_swapped -- +bob 2 + +-- !basic_filtered -- +2 +3 + +-- !basic_count_filtered -- +1 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 false -1 -2 -3 -4 -1.5 -2.5 -123.4567 char2 string2 040506 0C0D 2026-01-02 2026-01-02T01:02:03.456789 2026-01-02T09:02:03.456 [4, 5] {"k3":3} {"r_int":2, "r_string":"nested2"} +3 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !types_nested -- +1 3 1 1 2 1 nested1 +2 2 4 \N \N 2 nested2 +3 \N \N \N \N \N \N + +-- !part_all -- +1 p1a 20260101 +2 p1b 20260101 +3 p2a 20260102 +4 p3a 20260103 + +-- !part_grouped -- +20260101 2 +20260102 1 +20260103 1 + +-- !part_pruned -- +1 +2 + +-- !empty_count -- +0 + +-- !empty_rows -- + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out b/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out new file mode 100644 index 00000000000000..3e9656acc4a4b9 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out @@ -0,0 +1,50 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !basic_all -- +1 k1 1.5 +2 k2-updated 22.5 +4 k4 4.5 + +-- !basic_count -- +3 + +-- !basic_deleted_key -- +0 + +-- !basic_updated_key -- +1 + +-- !basic_stale_value -- +0 + +-- !basic_names -- +k1 +k2-updated +k4 + +-- !basic_reordered -- +1.5 k1 + +-- !basic_filtered -- +2 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !types_nested -- +1 3 1 1 2 1 nested1 +2 \N \N \N \N \N \N + +-- !part_all -- +1 q1a 20260101 +2 q1b-updated 20260101 +3 q2a 20260102 + +-- !part_grouped -- +20260101 2 +20260102 1 + +-- !part_pruned -- +1 q1a +2 q1b-updated + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out b/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out new file mode 100644 index 00000000000000..99b260d73c5e7c --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !log_rows -- +1 lake1 1.10 +2 lake2 2.20 +3 lake3 3.30 +4 lake4 4.40 +5 hot5 5.50 +6 hot6 6.60 + +-- !log_count -- +6 + +-- !log_sum -- +23.10 + +-- !cold_rows -- +1 cold1 +2 cold2 +3 cold3 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !part_rows -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 +4 lp1c 20260101 + +-- !part_tiered_only -- +3 + +-- !part_with_tail -- +1 +2 +4 + +-- !plain_count -- +3 + diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy index 9d1f9490d4cfa7..983dbccbaf472d 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -38,55 +38,42 @@ suite("test_fluss_catalog", "p0,external") { """ // --- the fixture database and its tables are visible ------------------- - // Every result is bound to a variable before it is chained on: in a Groovy - // command expression `sql """..."""​.collect {}` would collect over the - // string, not over the rows. - def databaseRows = sql """show databases from ${catalogName}""" - def databases = databaseRows.collect { it[0] } - assertTrue(databases.contains("fluss_test"), - "fluss_test missing from ${catalogName}: ${databases}") + // The listings are recorded rather than spot-checked: a table that appears + // out of nowhere is as much a bug as one that goes missing, and only the + // whole list catches the first kind. In particular a lake table must be + // listed ONCE, under its own name -- the $lake reader is a way of reading it, + // not a second table, and a listing that showed both would double every lake + // table for anything walking the schema. + order_qt_databases """show databases from ${catalogName}""" sql """switch ${catalogName}""" sql """use fluss_test""" - def tableRows = sql """show tables""" - def tables = tableRows.collect { it[0] } - for (String expected : ["log_basic", "log_types", "log_part", "log_empty", "pk_basic", "pk_types", - "pk_part", "lake_log", "lake_cold", "lake_types", "lake_part", "lake_pk"]) { - assertTrue(tables.contains(expected), "table ${expected} missing: ${tables}") - } - // A lake table is listed once, under its own name. The $lake reader is a way - // of reading it, not a second table, and a catalog listing that showed both - // would double every lake table for anything walking the schema. - assertTrue(tables.every { !it.toString().contains("\$") }, - "system tables leaked into show tables: ${tables}") + order_qt_tables """show tables""" // --- schema mapping ---------------------------------------------------- - // desc rows are [Field, Type, Null, Key, Default, Extra]. - def descLogBasic = sql """desc log_basic""" - assertEquals(["id", "name", "price"], descLogBasic.collect { it[0] }) - - // One column per fluss type the connector maps, plus the id column. A - // dropped or duplicated column shows up here before any query runs. - def descLogTypes = sql """desc log_types""" - assertEquals(["id", "f_boolean", "f_tinyint", "f_smallint", "f_int", "f_bigint", - "f_float", "f_double", "f_decimal", "f_char", "f_string", "f_binary", - "f_bytes", "f_date", "f_timestamp", "f_timestamp_ltz", "f_array", - "f_map", "f_row"], - descLogTypes.collect { it[0] }) + // desc is recorded UNSORTED: column order is part of what is being checked, + // and sorting the rows would throw it away. Types are pinned here too, so a + // type-mapping change shows up before any query runs. + qt_desc_log_basic """desc log_basic""" + + // One column per fluss type the connector maps, plus the id column. + qt_desc_log_types """desc log_types""" // The partition key is an ordinary column of the table, not a hidden one. - def descLogPart = sql """desc log_part""" - assertEquals(["id", "name", "dt"], descLogPart.collect { it[0] }) + qt_desc_log_part """desc log_part""" // Primary-key columns keep their position; the connector reports every // column as a key column, which is how Doris models external tables. - def descPkBasic = sql """desc pk_basic""" - assertEquals(["id", "name", "score"], descPkBasic.collect { it[0] }) + qt_desc_pk_basic """desc pk_basic""" // --- comments survive the metadata mapping ----------------------------- // Column comments live on the fluss schema, not on the row type: reading // the row type instead would silently drop every one of them. + // + // Not recorded into a .out on purpose: the statement carries the catalog's + // properties, including this environment's bootstrap address, so a recorded + // baseline would be tied to the machine that generated it. def createTableRows = sql """show create table log_basic""" def createTable = createTableRows[0][1].toString() assertTrue(createTable.contains("row id"), "column comment lost: ${createTable}") @@ -94,10 +81,11 @@ suite("test_fluss_catalog", "p0,external") { "table comment lost: ${createTable}") // --- refresh keeps the catalog usable ---------------------------------- + // Recorded again rather than compared with the listing above: the two blocks + // have to stay identical in the baseline, which is the same statement made + // in a way that also survives someone adding a fixture table. sql """refresh catalog ${catalogName}""" - def refreshedRows = sql """show tables""" - def tablesAfterRefresh = refreshedRows.collect { it[0] } - assertEquals(tables.sort(), tablesAfterRefresh.sort()) + order_qt_tables_after_refresh """show tables""" sql """switch internal""" sql """drop catalog ${catalogName}""" diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy index 59c4ca9108a230..d073272a9a6c5a 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy @@ -53,44 +53,43 @@ suite("test_fluss_lake_only", "p0,external") { // The connector is wired into the v2 file scanner only, and fuzzy sessions // randomize this variable. sql """set enable_file_scanner_v2 = true""" - - def scalarOf = { String query -> sql(query)[0][0].toString() } + // TIMESTAMP_LTZ renders through the session time zone, and the baselines below + // record what it rendered as. + sql """set time_zone = 'Asia/Shanghai'""" // --- the lake holds what was tiered, and nothing that came after --------- // lake_log got four rows before the tiering service was stopped and two // after. Reading the lake alone must return the first four: a $lake that // quietly fell back to the fluss read would return all six. - def lakeRows = sql """select id, name, price from lake_log\$lake order by id""" - assertEquals(4, lakeRows.size()) - assertEquals(["1", "lake1", "1.10"], lakeRows[0].collect { it.toString() }) - assertEquals(["2", "lake2", "2.20"], lakeRows[1].collect { it.toString() }) - assertEquals(["3", "lake3", "3.30"], lakeRows[2].collect { it.toString() }) - assertEquals(["4", "lake4", "4.40"], lakeRows[3].collect { it.toString() }) + order_qt_lake_rows """select id, name, price from lake_log\$lake""" // --- the three columns fluss adds to every lake table -------------------- // They belong to the lake table and not to the fluss one, which is the whole // reason the two are exposed as separate tables rather than one. - def systemColumns = sql """ + // + // Their VALUES are deliberately not recorded: which bucket a log row lands in + // is the writer's choice and __timestamp is a wall clock, so a baseline holding + // them would be rewritten every time the environment is rebuilt. What is + // recorded is that every row has all three within the range they must be in. + order_qt_system_columns """ select count(*) from lake_log\$lake where __bucket >= 0 and __bucket < 3 and __offset >= 0 and __timestamp is not null """ - assertEquals("4", systemColumns[0][0].toString()) - - def lakeSchema = sql """desc lake_log\$lake""" - def lakeColumnNames = lakeSchema.collect { it[0].toString() } - assertEquals(["id", "name", "price", "__bucket", "__offset", "__timestamp"], lakeColumnNames) + qt_desc_lake_log_lake """desc lake_log\$lake""" // The fluss table itself has none of them. - def flussColumnNames = sql("""desc lake_log""").collect { it[0].toString() } - assertEquals(["id", "name", "price"], flussColumnNames) + qt_desc_lake_log """desc lake_log""" // --- type parity between the two doors ---------------------------------- // The connector's fluss->Doris mapping has to equal fluss->paimon->Doris, or - // `tbl` and `tbl$lake` present two different schemas for one table. Until now - // that was checked by reading both mappings side by side; here the second one - // is the paimon connector actually running. + // `tbl` and `tbl$lake` present two different schemas for one table. Both + // schemas are recorded, and the equality is ALSO asserted here: a reader + // comparing two recorded blocks by eye is not what should be guarding an + // invariant this quiet. + qt_desc_lake_types """desc lake_types""" + qt_desc_lake_types_lake """desc lake_types\$lake""" def typesOf = { String table -> def result = [:] sql("""desc ${table}""").each { row -> result.put(row[0].toString(), row[1].toString()) } @@ -104,68 +103,51 @@ suite("test_fluss_lake_only", "p0,external") { } assertEquals(flussTypes.size() + 3, lakeTypes.size()) - // Parity of the values, not just of the declared types: row 1 of lake_types - // is compared against itself read the other way round, through fluss rather - // than paimon. Two decoders, one row, one set of literals. - def lakeTypeRow = sql """ - select count(*) from lake_types\$lake where id = 1 - and f_boolean = true - and f_tinyint = 1 and f_smallint = 2 and f_int = 3 and f_bigint = 4 - and f_float = cast(1.5 as float) and f_double = 2.5 - and f_decimal = 123.4567 - and f_char = 'char1' and f_string = 'string1' - and hex(f_binary) = '010203' and hex(f_bytes) = '0A0B' - and f_date = '2026-01-01' - -- Compared as text, not as a timestamp: an equality on a microsecond TIMESTAMP is pushed - -- into paimon and matches nothing there, while a range predicate on the same column and - -- the value itself are both right. That is the paimon connector's own behaviour (a plain - -- paimon catalog over this warehouse does the same), so pinning it here would assert - -- someone else's bug. The value is what this suite is about, and casting keeps the - -- comparison in Doris. - and cast(f_timestamp as string) = '2026-01-01 01:02:03.456789' - and f_timestamp_ltz is not null - and array_size(f_array) = 3 and f_array[1] = 1 and f_array[3] = 3 - and f_map['k1'] = 1 and f_map['k2'] = 2 - and struct_element(f_row, 'r_int') = 1 - and struct_element(f_row, 'r_string') = 'nested1' + // Parity of the values, not just of the declared types: the row is recorded + // here read through paimon, and the same row read through fluss is recorded in + // test_fluss_log_table -- two decoders, one row, and the two baselines have to + // agree column for column. + // + // BINARY and BYTES go through hex() for the same reason as everywhere else. Note + // that an equality predicate on the microsecond TIMESTAMP would match nothing: + // it is pushed into paimon, which does not find the row, while the value itself + // and range predicates on it are right. That is the paimon connector's own + // behaviour (a plain paimon catalog over this warehouse does the same), so it is + // not pinned here -- recording the value sidesteps it entirely. + order_qt_lake_types_row """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from lake_types\$lake """ - assertEquals("1", lakeTypeRow[0][0].toString()) // The all-NULL row was written after tiering stopped, so it is not here. - assertEquals("1", scalarOf("""select count(*) from lake_types\$lake""")) + order_qt_lake_types_count """select count(*) from lake_types\$lake""" // --- a table the lake holds in full -------------------------------------- - def coldRows = sql """select id, name from lake_cold\$lake order by id""" - assertEquals([["1", "cold1"], ["2", "cold2"], ["3", "cold3"]], - coldRows.collect { row -> row.collect { it.toString() } }) + order_qt_cold_rows """select id, name from lake_cold\$lake""" // --- partitioning survives the delegation -------------------------------- // The lake table is partitioned by the same column, so the partition value // has to come back with its own row and not with a neighbour's. - def partRows = sql """select id, name, dt from lake_part\$lake order by id""" - assertEquals(3, partRows.size()) - assertEquals(["1", "lp1a", "20260101"], partRows[0].collect { it.toString() }) - assertEquals(["2", "lp1b", "20260101"], partRows[1].collect { it.toString() }) - assertEquals(["3", "lp2a", "20260102"], partRows[2].collect { it.toString() }) + order_qt_part_rows """select id, name, dt from lake_part\$lake""" // Pruning is the sibling's, not fluss's: the predicate is pushed to the // paimon connector, which owns the plan for this table. - def prunedPart = sql """select id from lake_part\$lake where dt = '20260101' order by id""" - assertEquals(["1", "2"], prunedPart.collect { it[0].toString() }) + order_qt_part_pruned """select id from lake_part\$lake where dt = '20260101'""" // --- a primary-key table's lake is its merged state at the tiering point -- // Row 2 was updated before tiering, so the lake holds the update, not both // versions. Row 3's later update and row 1's delete came after and are absent, - // which is exactly how this differs from the fluss-only read of the same table. - def pkLakeRows = sql """select id, name from lake_pk\$lake order by id""" - assertEquals([["1", "lp1"], ["2", "lp2-lake"], ["3", "lp3"]], - pkLakeRows.collect { row -> row.collect { it.toString() } }) + // which is exactly how this differs from the fluss-only read of the same table + // (recorded in test_fluss_lake_pk, against this same fixture). + order_qt_pk_lake_rows """select id, name from lake_pk\$lake""" // --- projection and aggregation through the sibling ---------------------- - assertEquals("4", scalarOf("""select count(*) from lake_log\$lake""")) - assertEquals("11.00", scalarOf("""select sum(price) from lake_log\$lake""")) - def namesOnly = sql """select name from lake_log\$lake where id > 2 order by name""" - assertEquals(["lake3", "lake4"], namesOnly.collect { it[0].toString() }) + order_qt_lake_count """select count(*) from lake_log\$lake""" + order_qt_lake_sum """select sum(price) from lake_log\$lake""" + order_qt_lake_names """select name from lake_log\$lake where id > 2""" // --- tables with no lake ------------------------------------------------- // A table with no lake never offers the sub-table, so the name does not resolve @@ -182,11 +164,8 @@ suite("test_fluss_lake_only", "p0,external") { // $lake is a way to read a table, not a table of its own: it must not appear // in the catalog listing, or every tool that walks the schema would show each - // lake table twice. - def tableNames = sql("""show tables""").collect { it[0].toString() } - assertTrue(tableNames.contains("lake_log"), "lake_log missing from ${tableNames}") - assertTrue(tableNames.every { !it.contains("\$") }, - "system tables leaked into show tables: ${tableNames}") + // lake table twice. The recorded listing is what says so. + order_qt_tables """show tables""" sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy index 38d907d468e3a6..070a040bb4e52d 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy @@ -20,9 +20,10 @@ // Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are // static - this suite never writes, so no polling gate is needed. // -// Everything is asserted explicitly rather than through qt_ and a .out file. The -// values are the fixture's own literals, so an expectation that drifts from the -// fixture is a diff in one file rather than a regenerated baseline nobody reads. +// Results are recorded into the .out baseline; only what is NOT a result stays as +// an assertion in the code -- the EXPLAIN anchors, which say how the scan was +// planned, and the bounds on range counts, which cannot be exact because a fluss +// writer chooses its own buckets. suite("test_fluss_log_table", "p0,external") { String enabled = context.config.otherConfigs.get("enableFlussTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -49,8 +50,11 @@ suite("test_fluss_log_table", "p0,external") { // this variable, so pinning it is what keeps the suite from failing on half the // CI runs for a reason that has nothing to do with fluss. sql """set enable_file_scanner_v2 = true""" + // TIMESTAMP_LTZ renders through the session time zone, and the baseline below + // records what it rendered as. Without pinning it, the recorded value would be + // whatever the machine that generated the baseline happened to be set to. + sql """set time_zone = 'Asia/Shanghai'""" - def scalarOf = { String query -> sql(query)[0][0].toString() } def planOf = { String query -> def planRows = sql("""explain ${query}""") return planRows.collect { it[0].toString() }.join("\n") @@ -62,127 +66,60 @@ suite("test_fluss_log_table", "p0,external") { } // --- the whole table ---------------------------------------------------- - def basicRows = sql """select id, name, price from log_basic order by id""" - assertEquals(3, basicRows.size()) - assertEquals(["1", "alice", "10.10"], basicRows[0].collect { it.toString() }) - assertEquals(["2", "bob", "20.20"], basicRows[1].collect { it.toString() }) - assertEquals(["3", "carol", "30.30"], basicRows[2].collect { it.toString() }) + order_qt_basic_all """select id, name, price from log_basic""" // COUNT(*) projects no column at all: the scanner has to report how many rows it // read without returning one. A scanner that answered with an empty batch instead // would make an untouched-looking table out of a populated one. - assertEquals("3", scalarOf("""select count(*) from log_basic""")) + order_qt_basic_count """select count(*) from log_basic""" // --- projection and predicates ----------------------------------------- - def names = sql """select name from log_basic order by name""" - assertEquals(["alice", "bob", "carol"], names.collect { it[0].toString() }) + order_qt_basic_names """select name from log_basic""" // Columns asked for in an order other than the table's: the scanner resolves the // projection by name, so a positional shortcut anywhere would swap these two. - def swapped = sql """select name, id from log_basic where id = 2""" - assertEquals(1, swapped.size()) - assertEquals(["bob", "2"], swapped[0].collect { it.toString() }) + order_qt_basic_swapped """select name, id from log_basic where id = 2""" - def filtered = sql """select id from log_basic where price > 15.00 order by id""" - assertEquals(["2", "3"], filtered.collect { it[0].toString() }) - assertEquals("1", scalarOf("""select count(*) from log_basic where name = 'bob'""")) + order_qt_basic_filtered """select id from log_basic where price > 15.00""" + order_qt_basic_count_filtered """select count(*) from log_basic where name = 'bob'""" // --- every mapped type survives the round trip -------------------------- - // Asserted through predicates rather than by comparing rendered values: the - // rendering of a map or a struct is the display layer's business, while what this - // suite is about is whether the bytes fluss returned decoded into the right value. - assertEquals("3", scalarOf("""select count(*) from log_types""")) - - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 1 - and f_boolean = true - and f_tinyint = 1 - and f_smallint = 2 - and f_int = 3 - and f_bigint = 4 - and f_float = cast(1.5 as float) - and f_double = 2.5 - and f_decimal = 123.4567 - """)) - - // Fluss BYTES and BINARY map to a Doris string by default, so their content is - // compared as hex; a decoder that lost the length would return a prefix of this. - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 1 - and f_char = 'char1' - and f_string = 'string1' - and hex(f_binary) = '010203' - and hex(f_bytes) = '0A0B' - """)) - - // TIMESTAMP_LTZ is only checked for presence: its rendering depends on the session - // time zone, which is not what this suite is pinning. - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 1 - and f_date = '2026-01-01' - and f_timestamp = '2026-01-01 01:02:03.456789' - and f_timestamp_ltz is not null - """)) - - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 1 - and array_size(f_array) = 3 - and f_array[1] = 1 - and f_array[3] = 3 - and f_map['k1'] = 1 - and f_map['k2'] = 2 - and struct_element(f_row, 'r_int') = 1 - and struct_element(f_row, 'r_string') = 'nested1' - """)) - - // The second row is the negative-value one: a sign lost in decoding shows up here - // and nowhere else. - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 2 - and f_boolean = false - and f_tinyint = -1 - and f_smallint = -2 - and f_int = -3 - and f_bigint = -4 - and f_float = cast(-1.5 as float) - and f_double = -2.5 - and f_decimal = -123.4567 - and array_size(f_array) = 2 - """)) - - // The all-NULL row. Read through a null map that is off by one column, every value - // after it shifts, so this is checked column by column rather than by row count. - assertEquals("1", scalarOf(""" - select count(*) from log_types where id = 3 - and f_boolean is null and f_tinyint is null and f_smallint is null - and f_int is null and f_bigint is null and f_float is null - and f_double is null and f_decimal is null and f_char is null - and f_string is null and f_binary is null and f_bytes is null - and f_date is null and f_timestamp is null and f_timestamp_ltz is null - and f_array is null and f_map is null and f_row is null - """)) + // The three rows are recorded whole: the positive one, the negative one (a sign + // lost in decoding shows up nowhere else) and the all-NULL one (read through a + // null map that is off by one column, every value after it shifts). Recording + // them pins the rendering of map, struct and decimal as well, which is more than + // the decoding this suite is about -- a rendering change will land here and has + // to be re-recorded deliberately. + // + // BINARY and BYTES go through hex(): they map to a Doris string, and a raw byte + // in a recorded baseline is neither readable nor safely round-tripped. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from log_types + """ + + // The nested readers, asked for one element at a time: a struct that decoded into + // the right shape but the wrong field order still renders plausibly above. + order_qt_types_nested """ + select id, array_size(f_array), f_array[1], f_map['k1'], f_map['k2'], + struct_element(f_row, 'r_int'), struct_element(f_row, 'r_string') + from log_types + """ // --- partitioned table -------------------------------------------------- // The partition column is not read by the scanner: FE declares it and BE fills it - // in from each range. Checking it against the row it belongs to is what catches a + // in from each range. Recording it next to the row it belongs to is what catches a // partition value attached to the wrong split. - def partRows = sql """select id, name, dt from log_part order by id""" - assertEquals(4, partRows.size()) - assertEquals(["1", "p1a", "20260101"], partRows[0].collect { it.toString() }) - assertEquals(["2", "p1b", "20260101"], partRows[1].collect { it.toString() }) - assertEquals(["3", "p2a", "20260102"], partRows[2].collect { it.toString() }) - assertEquals(["4", "p3a", "20260103"], partRows[3].collect { it.toString() }) + order_qt_part_all """select id, name, dt from log_part""" // Projecting nothing but the partition column leaves the scanner with an empty // projection - it still has to report the row count for each range. - def perPartition = sql """select dt, count(*) from log_part group by dt order by dt""" - assertEquals(3, perPartition.size()) - assertEquals(["20260101", "2"], perPartition[0].collect { it.toString() }) - assertEquals(["20260102", "1"], perPartition[1].collect { it.toString() }) - assertEquals(["20260103", "1"], perPartition[2].collect { it.toString() }) + order_qt_part_grouped """select dt, count(*) from log_part group by dt""" - def prunedRows = sql """select id from log_part where dt = '20260101' order by id""" - assertEquals(["1", "2"], prunedRows.collect { it[0].toString() }) + order_qt_part_pruned """select id from log_part where dt = '20260101'""" // --- planning is visible in the plan ------------------------------------ def basicPlan = planOf("""select * from log_basic""") @@ -215,9 +152,8 @@ suite("test_fluss_log_table", "p0,external") { // --- a table that was never written to ---------------------------------- // Its buckets stop at offset 0, so planning emits no range at all. The empty answer // has to come from that, not from a scanner opened on an empty bucket. - assertEquals("0", scalarOf("""select count(*) from log_empty""")) - def emptyRows = sql """select id, name from log_empty""" - assertEquals(0, emptyRows.size()) + order_qt_empty_count """select count(*) from log_empty""" + order_qt_empty_rows """select id, name from log_empty""" def emptyPlan = planOf("""select * from log_empty""") // An engine that drops a split-less scan altogether is just as correct as one that // keeps the node, so what is pinned is that no range was planned either way. diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy index 1ca54f442bab52..af29f1f8a7d79b 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy @@ -29,10 +29,10 @@ // README), so these queries take that path rather than replaying the change log. // // Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are -// static - this suite never writes, so no polling gate is needed. Everything is -// asserted explicitly rather than through qt_ and a .out file: the values are the -// fixture's own literals, so an expectation that drifts from the fixture is a diff -// in one file rather than a regenerated baseline nobody reads. +// static - this suite never writes, so no polling gate is needed. Results are +// recorded into the .out baseline; what stays in the code is what is not a result: +// the EXPLAIN anchors saying how the scan was planned, and range-count bounds, +// which cannot be exact because key-to-bucket hashing is fluss's business. suite("test_fluss_pk_table", "p0,external") { String enabled = context.config.otherConfigs.get("enableFlussTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -59,8 +59,11 @@ suite("test_fluss_pk_table", "p0,external") { // this variable, so pinning it is what keeps the suite from failing on half the // CI runs for a reason that has nothing to do with fluss. sql """set enable_file_scanner_v2 = true""" + // TIMESTAMP_LTZ renders through the session time zone, and the baseline below + // records what it rendered as. Without pinning it, the recorded value would be + // whatever the machine that generated the baseline happened to be set to. + sql """set time_zone = 'Asia/Shanghai'""" - def scalarOf = { String query -> sql(query)[0][0].toString() } def planOf = { String query -> def planRows = sql("""explain ${query}""") return planRows.collect { it[0].toString() }.join("\n") @@ -75,118 +78,65 @@ suite("test_fluss_pk_table", "p0,external") { // The fixture inserts four rows, updates one and deletes one. Four writes, three // rows: id 2 carries its later value and id 3 is gone. Reading the change log // straight through would answer six rows here, all of them real records. - def basicRows = sql """select id, name, score from pk_basic order by id""" - assertEquals(3, basicRows.size()) - assertEquals(["1", "k1", "1.5"], basicRows[0].collect { it.toString() }) - assertEquals(["2", "k2-updated", "22.5"], basicRows[1].collect { it.toString() }) - assertEquals(["4", "k4", "4.5"], basicRows[2].collect { it.toString() }) + order_qt_basic_all """select id, name, score from pk_basic""" // COUNT(*) projects no column at all. On a primary-key table the count still has // to be the merged one: counting change log records would report six. - assertEquals("3", scalarOf("""select count(*) from pk_basic""")) + order_qt_basic_count """select count(*) from pk_basic""" - // The deleted key must be absent, not merely superseded. - assertEquals("0", scalarOf("""select count(*) from pk_basic where id = 3""")) - // And the updated key must appear once, not once per version. - assertEquals("1", scalarOf("""select count(*) from pk_basic where id = 2""")) - assertEquals("0", scalarOf("""select count(*) from pk_basic where name = 'k2'""")) + // The deleted key must be absent, not merely superseded; the updated key must + // appear once, not once per version, and never under its old value. + order_qt_basic_deleted_key """select count(*) from pk_basic where id = 3""" + order_qt_basic_updated_key """select count(*) from pk_basic where id = 2""" + order_qt_basic_stale_value """select count(*) from pk_basic where name = 'k2'""" // --- projection and predicates ----------------------------------------- // The merge needs the primary key even when the query does not ask for it, so the // reader adds it to what it fetches and projects it back out. A leaked key column // or a fetch-order projection shows up as the wrong values here. - def names = sql """select name from pk_basic order by name""" - assertEquals(["k1", "k2-updated", "k4"], names.collect { it[0].toString() }) + order_qt_basic_names """select name from pk_basic""" - def reordered = sql """select score, name from pk_basic where id = 1""" - assertEquals(["1.5", "k1"], reordered[0].collect { it.toString() }) + order_qt_basic_reordered """select score, name from pk_basic where id = 1""" - assertEquals("2", scalarOf("""select count(*) from pk_basic where score > 2.0""")) + order_qt_basic_filtered """select count(*) from pk_basic where score > 2.0""" // --- every mapped type, in the kv row format --------------------------- // Primary-key tables store rows in a different format from a log table's, so this - // repeats the type coverage rather than trusting the log suite for it. Asserted by - // predicate, not by rendering: how a decimal or a map prints is the display layer's - // business, and pinning it here would make this suite fail for the wrong reasons. - assertEquals("2", scalarOf("""select count(*) from pk_types""")) - - assertEquals("1", scalarOf(""" - select count(*) from pk_types where id = 1 - and f_boolean = true - and f_tinyint = 1 - and f_smallint = 2 - and f_int = 3 - and f_bigint = 4 - and f_float = cast(1.5 as float) - and f_double = 2.5 - and f_decimal = 123.4567 - """)) - - // Fluss BYTES and BINARY map to a Doris string by default, so their content is - // compared as hex; a decoder that lost the length would return a prefix of this. - assertEquals("1", scalarOf(""" - select count(*) from pk_types where id = 1 - and f_char = 'char1' - and f_string = 'string1' - and hex(f_binary) = '010203' - and hex(f_bytes) = '0A0B' - """)) - - // TIMESTAMP_LTZ is only checked for presence: its rendering depends on the session - // time zone, which is not what this suite is pinning. - assertEquals("1", scalarOf(""" - select count(*) from pk_types where id = 1 - and f_date = '2026-01-01' - and f_timestamp = '2026-01-01 01:02:03.456789' - and f_timestamp_ltz is not null - """)) - - assertEquals("1", scalarOf(""" - select count(*) from pk_types where id = 1 - and array_size(f_array) = 3 - and f_array[1] = 1 - and f_array[3] = 3 - and f_map['k1'] = 1 - and f_map['k2'] = 2 - and struct_element(f_row, 'r_int') = 1 - and struct_element(f_row, 'r_string') = 'nested1' - """)) - - // The all-NULL row. Read through a null map that is off by one column, every value - // after it shifts, so this is checked column by column rather than by row count. - assertEquals("1", scalarOf(""" - select count(*) from pk_types where id = 2 - and f_boolean is null and f_tinyint is null and f_smallint is null - and f_int is null and f_bigint is null and f_float is null - and f_double is null and f_decimal is null and f_char is null - and f_string is null and f_binary is null and f_bytes is null - and f_date is null and f_timestamp is null and f_timestamp_ltz is null - and f_array is null and f_map is null and f_row is null - """)) + // repeats the type coverage rather than trusting the log suite for it. Both rows + // are recorded whole -- the populated one and the all-NULL one, where a null map + // off by one column shifts every value after it. + // + // BINARY and BYTES go through hex(): they map to a Doris string, and a raw byte + // in a recorded baseline is neither readable nor safely round-tripped. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from pk_types + """ + + // The nested readers, asked for one element at a time: a struct that decoded into + // the right shape but the wrong field order still renders plausibly above. + order_qt_types_nested """ + select id, array_size(f_array), f_array[1], f_map['k1'], f_map['k2'], + struct_element(f_row, 'r_int'), struct_element(f_row, 'r_string') + from pk_types + """ // --- partitioned primary-key table -------------------------------------- // A partitioned primary-key table is snapshotted per partition, so its snapshots // have to be asked for per partition too; asking at table level resumes the change // log at another partition's offset. The partition column itself is not read by the - // scanner - FE declares it and BE fills it in from each range - so checking it - // against the row it belongs to is what catches a partition value on the wrong split. - def partRows = sql """select id, name, dt from pk_part order by dt, id""" - assertEquals(3, partRows.size()) - assertEquals(["1", "q1a", "20260101"], partRows[0].collect { it.toString() }) - assertEquals(["2", "q1b-updated", "20260101"], partRows[1].collect { it.toString() }) - assertEquals(["3", "q2a", "20260102"], partRows[2].collect { it.toString() }) + // scanner - FE declares it and BE fills it in from each range - so recording it + // next to the row it belongs to is what catches a partition value on the wrong split. + order_qt_part_all """select id, name, dt from pk_part""" // The delete landed in 20260102 and the update in 20260101: a merge that crossed // partitions would lose or resurrect one of them. - def perPartition = sql """select dt, count(*) from pk_part group by dt order by dt""" - assertEquals(2, perPartition.size()) - assertEquals(["20260101", "2"], perPartition[0].collect { it.toString() }) - assertEquals(["20260102", "1"], perPartition[1].collect { it.toString() }) - - def prunedRows = sql """select id, name from pk_part where dt = '20260101' order by id""" - assertEquals(2, prunedRows.size()) - assertEquals(["1", "q1a"], prunedRows[0].collect { it.toString() }) - assertEquals(["2", "q1b-updated"], prunedRows[1].collect { it.toString() }) + order_qt_part_grouped """select dt, count(*) from pk_part group by dt""" + + order_qt_part_pruned """select id, name from pk_part where dt = '20260101'""" // --- planning is visible in the plan ------------------------------------ def basicPlan = planOf("""select * from pk_basic""") diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy index b0a7240e1fb53e..d612c49cb04c9f 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy @@ -67,7 +67,6 @@ suite("test_fluss_union_log", "p0,external") { sql """use fluss_test""" sql """set enable_file_scanner_v2 = true""" - def scalarOf = { String query -> sql(query)[0][0].toString() } def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } def planOf = { String query -> return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") @@ -92,26 +91,25 @@ suite("test_fluss_union_log", "p0,external") { assertEquals(0, countIn(logPlan, "pkRanges")) // --- the seam: same rows down both paths --------------------------------- - def compareModes = { String query, int expectedRows -> + // The comparison stays in the code even though the results are recorded: it is + // not a hand-written expectation but the load-bearing check of this suite, and + // it says something no pair of recorded blocks does -- that two entirely + // different readers agreed on this table, row for row. + def compareModes = { String query -> def union = rowsOf("""${query}""") def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) - assertEquals(expectedRows, union.size(), "union read returned ${union.size()} rows for: ${query}") assertEquals(flussOnly, union, "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") - return union } // Four rows out of the lake and two out of the log, in one result. - def logRows = compareModes("select id, name, price from lake_log order by id", 6) - assertEquals(["1", "lake1", "1.10"], logRows[0]) - assertEquals(["4", "lake4", "4.40"], logRows[3]) - assertEquals(["5", "hot5", "5.50"], logRows[4]) - assertEquals(["6", "hot6", "6.60"], logRows[5]) + order_qt_log_rows """select id, name, price from lake_log""" + compareModes("select id, name, price from lake_log order by id") // Aggregates run over both halves; a half silently dropped shows up here even // when the ordered row list is not materialized. - assertEquals("6", scalarOf("""select count(*) from lake_log""")) - assertEquals("23.10", scalarOf("""select sum(price) from lake_log""")) + order_qt_log_count """select count(*) from lake_log""" + order_qt_log_sum """select sum(price) from lake_log""" // --- a table the lake already holds in full ------------------------------ // Nothing was written after tiering caught up, so every bucket's log range is @@ -121,84 +119,58 @@ suite("test_fluss_union_log", "p0,external") { assertTrue(coldPlan.contains("unionRead=yes"), "not a union read: ${coldPlan}") assertTrue(countIn(coldPlan, "lakeSplits") >= 1, "no lake splits: ${coldPlan}") assertEquals(0, countIn(coldPlan, "logRanges"), "cold table still has log ranges: ${coldPlan}") - compareModes("select id, name from lake_cold order by id", 3) + order_qt_cold_rows """select id, name from lake_cold""" + compareModes("select id, name from lake_cold order by id") // --- every type crosses the seam ----------------------------------------- - // Row 1 comes back through paimon, row 2 through the fluss log: two decoders - // feeding one result set, which is also what pins the mapping parity on real - // values rather than on declared types. - compareModes("select id from lake_types order by id", 2) - assertEquals("1", scalarOf(""" - select count(*) from lake_types where id = 1 - and f_boolean = true - and f_tinyint = 1 and f_smallint = 2 and f_int = 3 and f_bigint = 4 - and f_float = cast(1.5 as float) and f_double = 2.5 - and f_decimal = 123.4567 - and f_char = 'char1' and f_string = 'string1' - and hex(f_binary) = '010203' and hex(f_bytes) = '0A0B' - and f_date = '2026-01-01' - -- Cast rather than compared as a timestamp: this row comes from the lake half, and an - -- equality on a microsecond TIMESTAMP pushed into paimon matches nothing there. See the - -- same note in test_fluss_lake_only: it is the paimon connector's behaviour, reproducible - -- through a plain paimon catalog, and not something this suite should pin. - and cast(f_timestamp as string) = '2026-01-01 01:02:03.456789' - and f_timestamp_ltz is not null - and array_size(f_array) = 3 and f_array[1] = 1 and f_array[3] = 3 - and f_map['k1'] = 1 and f_map['k2'] = 2 - and struct_element(f_row, 'r_int') = 1 - and struct_element(f_row, 'r_string') = 'nested1' - """)) - // The all-NULL row arrived after tiering stopped, so it is the log half's. - // Checked column by column: a null map read one column off shifts everything - // after it, and a row count would not notice. - assertEquals("1", scalarOf(""" - select count(*) from lake_types where id = 2 - and f_boolean is null and f_tinyint is null and f_smallint is null - and f_int is null and f_bigint is null and f_float is null - and f_double is null and f_decimal is null and f_char is null - and f_string is null and f_binary is null and f_bytes is null - and f_date is null and f_timestamp is null and f_timestamp_ltz is null - and f_array is null and f_map is null and f_row is null - """)) + // Row 1 comes back through paimon, row 2 (the all-NULL one, written after + // tiering stopped) through the fluss log: two decoders feeding one result set, + // which is what pins the mapping parity on real values rather than on declared + // types. A null map read one column off shifts every value after it, and only + // recording the row column by column notices. + // + // BINARY and BYTES go through hex() as everywhere else. An equality predicate on + // the microsecond TIMESTAMP would match nothing for the lake half -- pushed into + // paimon, it finds no row, which is the paimon connector's own behaviour and not + // something this suite should pin; recording the value sidesteps it. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from lake_types + """ + compareModes("select id from lake_types order by id") // --- partitioning, where the two halves prune differently ---------------- // The fluss half is given the partitions the engine pruned to; the paimon half // ignores that list and prunes on the pushed-down predicate instead. Both have - // to land on the same partition, which is what these two assertions separate: - // one partition has a log tail, the other is served entirely from the lake. - def partRows = compareModes("select id, name, dt from lake_part order by id", 4) - assertEquals(["1", "lp1a", "20260101"], partRows[0]) - assertEquals(["3", "lp2a", "20260102"], partRows[2]) - assertEquals(["4", "lp1c", "20260101"], partRows[3]) + // to land on the same partition: one partition here has a log tail, the other is + // served entirely from the lake. + order_qt_part_rows """select id, name, dt from lake_part""" + compareModes("select id, name, dt from lake_part order by id") def tieredPartPlan = planOf("""select * from lake_part where dt = '20260102'""") assertTrue(tieredPartPlan.contains("unionRead=yes"), "not a union read: ${tieredPartPlan}") assertEquals(0, countIn(tieredPartPlan, "logRanges"), "a fully tiered partition still has log ranges: ${tieredPartPlan}") - assertEquals(["3"], rowsOf("""select id from lake_part where dt = '20260102'""").collect { it[0] }) + order_qt_part_tiered_only """select id from lake_part where dt = '20260102'""" def tailPartPlan = planOf("""select * from lake_part where dt = '20260101'""") assertEquals(1, countIn(tailPartPlan, "logRanges"), "the partition with a tail lost its log range: ${tailPartPlan}") - assertEquals(["1", "2", "4"], - rowsOf("""select id from lake_part where dt = '20260101' order by id""").collect { it[0] }) - - // --- what is not supported yet fails loudly ------------------------------ - // Merging a lake with a change log BY KEY is not implemented. The refusal has - // to name the primary key and say what reading with the lake switched off - // would and would not give, because that fallback returns a partial table for - // a primary-key table rather than the whole one. + order_qt_part_with_tail """select id from lake_part where dt = '20260101'""" + + // --- a primary-key table is not a log table ------------------------------ + // Its halves have to be merged BY KEY, which is not implemented, so `required` + // refuses it rather than falling back. What such a table DOES read as lives in + // test_fluss_lake_pk, together with the baseline that merge will have to + // reproduce. test { sql """select * from lake_pk""" - exception "primary-key" + exception "not implemented yet" } - // With the lake switched off the same table reads as the fluss-only merged - // view: row 2's pre-tiering update, row 3's post-tiering one, row 1 deleted. - def pkFlussOnly = rowsOf( - """select id, name from ${flussOnlyCatalog}.fluss_test.lake_pk order by id""") - assertEquals([["2", "lp2-lake"], ["3", "lp3-hot"]], pkFlussOnly) - // --- required does not mean "every table has a lake" --------------------- // A table with no lake at all is not an error in required mode: there is // nothing to fall back FROM. Only a lake table whose snapshot cannot be read @@ -206,7 +178,7 @@ suite("test_fluss_union_log", "p0,external") { def plainPlan = planOf("""select * from log_basic""") assertTrue(plainPlan.contains("flussScan: unionRead=no"), "unexpected union read: ${plainPlan}") assertEquals(0, countIn(plainPlan, "lakeSplits")) - assertEquals("3", scalarOf("""select count(*) from log_basic""")) + order_qt_plain_count """select count(*) from log_basic""" sql """switch internal""" sql """drop catalog if exists ${unionCatalog}""" From 212f086d5838a38298dd475eeca6196c64610f8d Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 17:35:58 +0800 Subject: [PATCH 26/35] [feat](paimon) Say which bucket a scan range came from A scan range this connector plans is opaque about its origin: the JNI arm carries a serialized split and nothing else, the native arm a file path and a byte interval. That is fine while the only reader is BE, which just reads what it is handed. It stops being fine once another connector plans splits here on behalf of its own table. The fluss connector does exactly that: a fluss table tiered into paimon keeps a bucket-identical layout, and reading it means pairing the lake data of bucket b with the log tail of bucket b that has not been tiered yet. Nothing on the range says b. Parsing it out of the data-file path would work only on the native arm and only by depending on this connector's directory layout. So carry it: paimon.bucket = DataSplit.bucket(), on the native and JNI arms alike -- which BE reader a split lands on is a session-level escape hatch the sibling does not control, and it must not change what the sibling can learn. FE-only; populateRangeParams does not forward it, so BE sees nothing new. Two ranges deliberately do NOT carry it. The collapsed COUNT(*) range stands for the splits of every bucket, so any single number on it would be a lie. A non-DataSplit system split has no bucket at all. A consumer that needs the binding must fail loud on an absent bucket rather than read it as "no state for this bucket" -- that reading turns a broken contract into duplicated rows. The fixture is two-bucket on purpose: with one bucket every range reads "0" and a hard-coded constant passes. Four mutations checked red -- constant bucket on the native arm, no bucket on the JNI arm, a bucket on the system split, a bucket on the count range. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../paimon/PaimonScanPlanProvider.java | 25 +- .../connector/paimon/PaimonScanRange.java | 21 ++ .../paimon/PaimonScanPlanProviderTest.java | 20 +- .../paimon/PaimonScanRangeBucketTest.java | 296 ++++++++++++++++++ 4 files changed, 344 insertions(+), 18 deletions(-) create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index ef5e4650d88f9c..3f6213ec339cc6 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -761,7 +761,8 @@ private List planScanInternal( (optDeletionFiles.isPresent() && i < optDeletionFiles.get().size()) ? optDeletionFiles.get().get(i) : null; ranges.addAll(buildNativeRanges(file, deletionFile, defaultFileFormat, - partitionValues, vendedToken, effectiveSplitSize, weightDenominator)); + partitionValues, vendedToken, effectiveSplitSize, weightDenominator, + dataSplit.bucket())); } } else { // JNI reader path @@ -798,7 +799,8 @@ private List planScanInternal( */ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, String defaultFileFormat, Map partitionValues, - Map vendedToken, long start, long length, long weightDenominator) { + Map vendedToken, long start, long length, long weightDenominator, + int bucket) { String fileFormat = getFileFormatBySuffix(file.path()).orElse(defaultFileFormat); // FIX-A1: native sub-split FE weight = the sub-range byte length, + the deletion-vector length when // attached (legacy PaimonSplit(LocationPath,...).selfSplitWeight = length, setDeletionFile += DV). @@ -814,7 +816,8 @@ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, .partitionValues(partitionValues) .selfSplitWeight(selfSplitWeight) .targetSplitSize(weightDenominator) - .schemaId(file.schemaId()); + .schemaId(file.schemaId()) + .bucket(bucket); if (deletionFile != null) { builder.deletionFile( normalizeUri(deletionFile.path(), vendedToken), @@ -836,11 +839,12 @@ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, */ List buildNativeRanges(RawFile file, DeletionFile deletionFile, String defaultFileFormat, Map partitionValues, - Map vendedToken, long targetSplitSize, long weightDenominator) { + Map vendedToken, long targetSplitSize, long weightDenominator, + int bucket) { List result = new ArrayList<>(); for (long[] offset : computeFileSplitOffsets(file.length(), targetSplitSize)) { result.add(buildNativeRange(file, deletionFile, defaultFileFormat, - partitionValues, vendedToken, offset[0], offset[1], weightDenominator)); + partitionValues, vendedToken, offset[0], offset[1], weightDenominator, bucket)); } return result; } @@ -1372,13 +1376,18 @@ private PaimonScanRange buildJniScanRange(Split split, String defaultFileFormat, String fileFormat = isDataSplit ? dataSplitFileFormat((DataSplit) split, defaultFileFormat) : defaultFileFormat; - return new PaimonScanRange.Builder() + PaimonScanRange.Builder builder = new PaimonScanRange.Builder() .fileFormat(fileFormat) .paimonSplit(serializedSplit) .partitionValues(partitionValues) .selfSplitWeight(splitWeight) - .targetSplitSize(weightDenominator) - .build(); + .targetSplitSize(weightDenominator); + if (isDataSplit) { + // Same bucket property as the native arm: which reader BE ends up using must not change + // what a sibling connector can learn about the split (see PaimonScanRange's props). + builder.bucket(((DataSplit) split).bucket()); + } + return builder.build(); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java index e6097aae4f5920..2a237762115741 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java @@ -91,6 +91,19 @@ private PaimonScanRange(Builder builder) { if (builder.rowCount != null) { props.put("paimon.row_count", String.valueOf(builder.rowCount)); } + // FE-ONLY (never reaches BE, see populateRangeParams): the paimon bucket this range's data + // belongs to, = DataSplit.bucket(). Read by a sibling connector that plans paimon splits on + // behalf of its own table and has to line them up with its own per-bucket state — today the + // fluss connector, whose lake half is planned here (it binds a fluss log tail to the lake + // splits of the SAME bucket; a lake table tiered from fluss has bucket-identical layout). + // Set on every DataSplit-backed range, native and JNI alike. NOT set on the collapsed + // COUNT(*) range (it stands for splits from many buckets, so any single number would be a + // lie) nor on a non-DataSplit system split (no bucket exists). Consumers must fail loud when + // it is absent on a range they expected to bind — silently treating that as "no state for + // this bucket" is a wrong-results bug, not a degradation. + if (builder.bucket != null) { + props.put("paimon.bucket", String.valueOf(builder.bucket)); + } // FIX-A3: emit the self-split-weight for every JNI split, incl. weight 0. Legacy // PaimonScanNode.setPaimonParams:274 sets it unconditionally on the JNI branch (never on // native); the old `selfSplitWeight > 0` gate was a buggy is-set proxy that dropped a genuine @@ -307,6 +320,9 @@ public static class Builder { // COUNT pushdown private Long rowCount; + // Bucket of the backing DataSplit; null for splits that have none (see the props comment). + private Integer bucket; + public Builder path(String path) { this.path = path; return this; @@ -369,6 +385,11 @@ public Builder rowCount(long rowCount) { return this; } + public Builder bucket(int bucket) { + this.bucket = bucket; + return this; + } + public PaimonScanRange build() { return new PaimonScanRange(this); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 1c3dc7f6460ff3..3a1a6442a0bd42 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -351,7 +351,7 @@ public void nativeRangeNormalizesBothDataAndDeletionVectorPaths() { "oss://bkt/warehouse/db/t/index/dv-0.index", 8L, 16L, 4L); PaimonScanRange range = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // WHY: BE's scheme-dispatched S3 file factory only opens canonical s3://. An un-normalized // oss:// DATA-file path fails the native ORC/Parquet read outright; an un-normalized oss:// DV @@ -376,7 +376,7 @@ public void nativeRangeWithoutDeletionVectorNormalizesOnlyDataPath() { PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", - Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // WHY: a DV-less native split must still normalize its data-file path and must NOT emit a DV // descriptor. MUTATION: emitting a deletion_file for a null DV, or skipping data normalization -> red. @@ -396,7 +396,7 @@ public void nativeRangeWithoutContextPreservesRawPath() { PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", - Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // MUTATION: NPE on null context, or fabricating a normalized path from nothing -> red. Assertions.assertEquals("oss://bkt/a/part-0.parquet", range.getPath().orElse(null)); @@ -421,7 +421,7 @@ public void buildNativeRangeThreadsVendedTokenToBothPaths() { "oss://bkt/warehouse/db/t/index/dv-0.index", 8L, 16L, 4L); PaimonScanRange range = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), vendedToken, 0L, 100L, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), vendedToken, 0L, 100L, 64L * 1024 * 1024, 0); // WHY: the engine seam normalizes against the VENDED map (the REST static map is empty). If the // connector dropped the token (reverting to the 1-arg seam) or substituted an empty map, a REST @@ -1733,7 +1733,7 @@ public void buildNativeRangesAttachesSameDeletionVectorToEverySubRange() { long target = Math.max(1L, file.length() / 3); // force the file to sub-split into >=2 ranges List ranges = provider.buildNativeRanges( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), target, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), target, 64L * 1024 * 1024, 0); // WHY: the load-bearing correctness claim of FIX-NATIVE-SUBSPLIT — a paimon deletion vector is a // bitmap of GLOBAL file row positions, so EVERY sub-range of a DV-bearing file must carry the @@ -1761,7 +1761,7 @@ public void buildNativeRangesKeepsFileWholeWhenTargetNonPositive() { RawFile file = parquetRawFile("oss://bkt/a/part-0.parquet"); List ranges = provider.buildNativeRanges( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L * 1024 * 1024); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L * 1024 * 1024, 0); Assertions.assertEquals(1, ranges.size(), "a non-positive target (COUNT(*) pushdown) must keep the file as one whole-file range"); @@ -2572,14 +2572,14 @@ public void buildNativeRangeSetsProportionalWeightFromLengthAndDv() { DeletionFile dv = new DeletionFile("/data/dv-0.index", 8L, 16L, 4L); PaimonScanRange withDv = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L, 64 * MB); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L, 64 * MB, 0); Assertions.assertEquals(64L + dv.length(), withDv.getSelfSplitWeight(), "native weight = sub-range length + the deletion-vector length"); Assertions.assertEquals(64 * MB, withDv.getTargetSplitSize(), "native range must carry the weight denominator"); PaimonScanRange noDv = provider.buildNativeRange( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 70L, 64 * MB); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 70L, 64 * MB, 0); Assertions.assertEquals(70L, noDv.getSelfSplitWeight(), "a DV-less native range weight is just the sub-range length"); } @@ -2597,7 +2597,7 @@ public void buildNativeRangesThreadsDenominatorDistinctFromFileSplitTarget() { List ranges = provider.buildNativeRanges( file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), - fileSplitTarget, denominator); + fileSplitTarget, denominator, 0); Assertions.assertEquals( PaimonScanPlanProvider.computeFileSplitOffsets(file.length(), fileSplitTarget).size(), @@ -2620,7 +2620,7 @@ public void buildNativeRangesCarriesDenominatorEvenWhenFileSplitSizeZero() { RawFile file = parquetRawFile("/data/part-0.parquet"); List ranges = provider.buildNativeRanges( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64 * MB); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64 * MB, 0); Assertions.assertEquals(1, ranges.size(), "a non-positive target keeps the file whole"); Assertions.assertEquals(64 * MB, ranges.get(0).getTargetSplitSize(), diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java new file mode 100644 index 00000000000000..24de90b2ad4571 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Pins the {@code paimon.bucket} scan-range property (P4-2-a0). + * + *

WHY this property exists: a sibling connector can plan paimon splits on behalf of its OWN table + * and then has to line each split up with its own per-bucket state. Today that sibling is the fluss + * connector: a fluss table tiered into paimon keeps bucket-identical layout, and the fluss connector + * binds the un-tiered log tail of bucket b to the lake splits of bucket b. Without the + * bucket on the range there is nothing in a {@link PaimonScanRange} that says which bucket it came + * from (the JNI arm carries only an opaque serialized split; the native arm only a file path), so the + * sibling would have to fall back to whole-table binding — correct but wasteful — or parse the + * sibling's internal directory layout, which the JNI arm does not even expose. + * + *

The fixture is deliberately MULTI-bucket: with a single bucket every range would carry "0" and a + * hard-coded constant would pass. Each test therefore asserts the ranges reproduce the split-side + * bucket SET, which a constant cannot. + */ +public class PaimonScanRangeBucketTest { + + /** + * A two-bucket PK table with rows in BOTH buckets. PK {@code id} hashes into + * {@code bucket = hash(id) % 2}; ids 1..8 cover both buckets for paimon's hash function. + */ + private static Table createTwoBucketTable(Catalog catalog) throws Exception { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "t"); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("val", DataTypes.BIGINT()) + .primaryKey("id") + .option("bucket", "2") + .build(), false); + Table table = catalog.getTable(id); + + BatchWriteBuilder wb = table.newBatchWriteBuilder(); + try (BatchTableWrite write = wb.newWrite()) { + for (int i = 1; i <= 8; i++) { + write.write(GenericRow.of(i, (long) i * 100)); + } + List messages = write.prepareCommit(); + try (BatchTableCommit commit = wb.newCommit()) { + commit.commit(messages); + } + } + return table; + } + + /** The buckets paimon's own read plan reports — the reference the ranges must reproduce. */ + private static Set planBuckets(Table table) throws Exception { + Set buckets = new TreeSet<>(); + for (Split s : table.newReadBuilder().newScan().plan().splits()) { + if (s instanceof DataSplit) { + buckets.add(((DataSplit) s).bucket()); + } + } + return buckets; + } + + /** The buckets the planned ranges claim, as ints. Fails the test if any range omits the property. */ + private static Set rangeBuckets(List ranges) { + Set buckets = new TreeSet<>(); + for (ConnectorScanRange r : ranges) { + String bucket = r.getProperties().get("paimon.bucket"); + Assertions.assertNotNull(bucket, + "every DataSplit-backed range must carry paimon.bucket; missing on " + r); + buckets.add(Integer.parseInt(bucket)); + } + return buckets; + } + + private static PaimonScanPlanProvider providerFor(Table table) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + return new PaimonScanPlanProvider(Collections.emptyMap(), ops); + } + + private static PaimonTableHandle handleFor(String tableName) { + return new PaimonTableHandle("db", tableName, + Collections.emptyList(), Collections.emptyList()); + } + + @Test + public void nativeRangesCarryTheBucketOfTheSplitTheyCameFrom(@TempDir Path warehouse) + throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Set expected = planBuckets(table); + Assertions.assertTrue(expected.size() >= 2, + "fixture precondition: the table must really span >=2 buckets, got " + expected); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "the fixture must plan at least one range"); + for (ConnectorScanRange r : ranges) { + Assertions.assertTrue(((PaimonScanRange) r).isNativeReadRange(), + "fixture precondition: this arm must exercise the NATIVE range builder"); + } + // WHY: a native range is one sub-range of one raw file of one DataSplit, so the bucket has + // to be threaded down from the DataSplit loop through buildNativeRanges/buildNativeRange. + // MUTATION: hard-coding 0 (or dropping .bucket() from the native builder) -> {0} instead of + // {0, 1} -> red; the multi-bucket fixture is what makes the constant detectable. + Assertions.assertEquals(expected, rangeBuckets(ranges), + "native ranges must reproduce exactly the buckets paimon's own plan reports"); + } + } + + @Test + public void jniRangesCarryTheBucketOfTheSplitTheyCameFrom(@TempDir Path warehouse) + throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Set expected = planBuckets(table); + Assertions.assertTrue(expected.size() >= 2, + "fixture precondition: the table must really span >=2 buckets, got " + expected); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")), + ConnectorScanRequest.builder(handleFor("t"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "the fixture must plan at least one range"); + for (ConnectorScanRange r : ranges) { + Assertions.assertTrue(r.getProperties().containsKey("paimon.split"), + "fixture precondition: this arm must exercise the JNI range builder"); + } + // WHY: which BE reader a split ends up on (native vs JNI, a session-level escape hatch the + // sibling does not control) must not change what the sibling can learn about the split. + // If only the native arm carried the bucket, turning on force_jni_scanner would silently + // break the sibling's binding. MUTATION: setting .bucket() only on the native arm -> the + // rangeBuckets assertNotNull fires -> red. + Assertions.assertEquals(expected, rangeBuckets(ranges), + "JNI ranges must reproduce exactly the buckets paimon's own plan reports"); + } + } + + @Test + public void collapsedCountRangeCarriesNoBucket(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Assertions.assertTrue(planBuckets(table).size() >= 2, + "fixture precondition: >=2 buckets, so the collapse really does span buckets"); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t"), noColumns()) + .countPushdown(true).build()); + + // WHY: the count collapse folds the splits of ALL buckets into ONE range carrying the summed + // total, so no single bucket number is true of it. Stamping the representative split's bucket + // would hand a sibling a range that claims to be bucket b while actually standing for every + // bucket — it would suppress/join against the wrong state. Absent is the honest answer, and + // the sibling is required to fail loud rather than guess (it never forwards count pushdown, + // so it must never see one of these). + // MUTATION: adding .bucket() to buildCountRange -> the count range carries one -> red. + int countRanges = 0; + for (ConnectorScanRange r : ranges) { + if (r.getProperties().containsKey("paimon.row_count")) { + ++countRanges; + Assertions.assertFalse(r.getProperties().containsKey("paimon.bucket"), + "the collapsed count range spans every bucket, so it must claim none"); + } + } + Assertions.assertEquals(1, countRanges, + "fixture precondition: count pushdown must produce exactly one collapsed range"); + } + } + + @Test + public void systemTableSplitCarriesNoBucket(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + createTwoBucketTable(catalog); + Table snapshots = catalog.getTable(Identifier.create("db", "t$snapshots")); + + List ranges = providerFor(snapshots).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t$snapshots"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "a snapshots system table must plan >=1 range"); + // WHY: a system-table split is not a DataSplit and has no bucket at all — fabricating one + // (say 0) would be a lie a sibling could act on. MUTATION: setting .bucket() unconditionally + // in buildJniScanRange (dropping the isDataSplit gate) -> red. This also documents the shape + // the sibling must reject: it plans only data reads, so a bucket-less range reaching its + // wrapper means the contract broke. + for (ConnectorScanRange r : ranges) { + Assertions.assertFalse(r.getProperties().containsKey("paimon.bucket"), + "a non-DataSplit system split has no bucket, so it must not claim one"); + } + } + } + + private static List noColumns() { + return Collections.emptyList(); + } + + private static ConnectorSession sessionWithProps(Map sessionProps) { + return new ConnectorSession() { + @Override + public String getQueryId() { + return "q"; + } + + @Override + public String getUser() { + return "u"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public long getCatalogId() { + return 0; + } + + @Override + public String getCatalogName() { + return "c"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + + @Override + public Map getSessionProperties() { + return sessionProps; + } + }; + } +} From 3285a8efc3aad8fa7f2f67bcf5a0e3784ed1fe39 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 17:54:32 +0800 Subject: [PATCH 27/35] [test](fluss) Drive the reader from a projection BE built itself Reading a tiered primary-key table means BE has to read the key columns of the log tail whether or not the query projects them -- they are what tells it which lake rows a newer log record has already replaced. That projection does not come from FE as slot descriptors the way every projection so far has; BE assembles it from names and types. Whether the reader accepts such a projection was an open question, and the answer decides the shape of the whole feature: if it does not, the key columns have to be driven straight into the JNI bridge as parameter strings instead. It does. The reader reads nothing off the columns but their name, type and partition flag, and both strings the Java scanner projects by are derived from that -- so a projection with no query behind it drives it the same as a planned one. Pins the two derived strings and the block C++ receives the columns into, because nothing between here and a wrong-columns read would notice. Checked red by skipping the derivation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../format_v2/jni/fluss_jni_reader_test.cpp | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/be/test/format_v2/jni/fluss_jni_reader_test.cpp b/be/test/format_v2/jni/fluss_jni_reader_test.cpp index 7f31e9147de9fe..2707c780b3aede 100644 --- a/be/test/format_v2/jni/fluss_jni_reader_test.cpp +++ b/be/test/format_v2/jni/fluss_jni_reader_test.cpp @@ -189,5 +189,46 @@ TEST(FlussJniReaderTest, StillReadsAPartitionColumnTheRangeDidNotCarry) { EXPECT_EQ(columns[1].output_index, 1); } +// A reader whose projection nobody planned. +// +// Every projection so far arrived from FE as slot descriptors: the query asked for these columns, +// the engine turned them into ColumnDefinitions, the reader read them. A union read of a primary-key +// table needs one that did not come from a query at all - to suppress the lake rows a newer log +// record has replaced, BE first has to read the key columns of that log tail, and those keys are +// needed whether or not the query happens to project them. +// +// What this pins is that a projection assembled here, from names and types alone, drives the reader +// exactly like a planned one: the scanner is asked for those columns, in that order, with those +// types. Everything downstream (the JNI block template, and the Java scanner's own projection) is +// derived from this pair of strings, so if they are right the rest follows. +TEST(FlussJniReaderTest, ReadsAProjectionBuiltHereRatherThanPlannedByFe) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + // No slot descriptors, no tuple, no query: three key columns named and typed on the spot. + std::vector synthetic_keys { + make_column("id", std::make_shared(), false), + make_column("name", std::make_shared(), false), + make_column("amount", std::make_shared(), false)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(synthetic_keys)).ok()); + + ASSERT_TRUE(reader.build_jni_columns(&reader._jni_columns).ok()); + ASSERT_EQ(reader._jni_columns.size(), 3); + reader._prepare_jni_scanner_schema(); + + // The two strings the Java scanner projects by. A synthetic projection that produced anything + // else here would read the wrong columns, or fail to open, at the first union read of a + // primary-key table - and nothing between here and there would notice. + EXPECT_EQ(reader._scanner_params["required_fields"], "id,name,amount"); + EXPECT_EQ(reader._scanner_params["columns_types"], "int#string#bigint"); + + // The block C++ receives them into. Its width and types must match the projection, since + // finalize_jni_block moves these columns into the output block by position. + ASSERT_EQ(reader._jni_block_template.columns(), 3); + EXPECT_EQ(reader._jni_block_template.get_by_position(0).name, "id"); + EXPECT_EQ(reader._jni_block_template.get_by_position(2).name, "amount"); + EXPECT_TRUE(reader._jni_block_template.get_by_position(2).type->equals(DataTypeInt64 {})); +} + } // namespace } // namespace doris::format::fluss From cc317fec37b229e0a3fb2e6481380072479a84e3 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 19:35:25 +0800 Subject: [PATCH 28/35] [feat](connector) Let a connector name the columns its reader must read A connector whose BE-side reader merges or suppresses rows by key needs that key read whether or not the query selected it. Doris already keeps those columns for its own aggregate and merge-on-read unique-key tables -- preserveExtraStorageKeySlots, four lines above where the scan's slots are pruned -- for exactly that reason. A plugin connector had no way to say the same thing, and BE cannot read a column the plan never asked for. So ask it: getMustReadColumns, answered per scan, empty by default, so nothing changes for a connector that needs only what the query projects. The answer arrives during plan translation, after the scan node is initialized and before splits are planned, and widens the scan's tuple only -- the project above it already has its own output tuple, so the column is read and then dropped rather than returned. The question goes through the same memoized provider that will plan the splits, because the two have to come from one decision: a connector that answers "no extra columns" here and then plans a read that needs them leaves BE looking for a column that is not in the projection. A name that matches no slot fails the query and says which name, rather than being skipped -- skipping turns a disagreement about the table into silently wrong rows. Checked red by six mutations: dropping the branch, skipping unknown names, stopping after the first match, dropping the null answer guard, resolving a fresh provider to ask, and a non-empty SPI default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../api/scan/ConnectorScanPlanProvider.java | 37 ++++ ...orScanPlanProviderMustReadColumnsTest.java | 79 ++++++++ .../datasource/scan/PluginDrivenScanNode.java | 23 +++ .../translator/PhysicalPlanTranslator.java | 39 ++++ ...uginDrivenScanNodeMustReadColumnsTest.java | 129 +++++++++++++ ...ysicalPlanTranslatorMustReadSlotsTest.java | 169 ++++++++++++++++++ 6 files changed, 476 insertions(+) create mode 100644 fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java index acd44d5dfc9136..a13af026a759b0 100644 --- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; /** * Plans the set of scan ranges (splits) needed to read a connector table. @@ -148,6 +149,42 @@ default TFileCompressType adjustFileCompressType(TFileCompressType inferred) { return inferred; } + /** + * The columns BE must READ for this scan even when the query references none of them, by Doris-side + * column name. The engine keeps their slots in the scan's tuple instead of pruning them away; the + * projection above the scan still removes them from the query's output, so the answer changes what is + * read, never what is returned. + * + *

This exists for a connector whose BE-side reader needs a column to produce CORRECT ROWS rather than + * to answer the query — a merge key, a suppression key, a row identity. Doris does the same thing for its + * own aggregate / merge-on-read unique-key tables ({@code PhysicalPlanTranslator.preserveExtraStorageKeySlots}): + * BE merges by key whether or not the user selected the key. Trino has no counterpart because its + * connectors own the page source and can add such columns privately; here the reader is BE, so the columns + * have to reach it through the plan.

+ * + *

Answer per SCAN, not per table: a connector that only sometimes needs the column (e.g. only when it + * decides to combine two sources) must return it only for those scans, and must reach the SAME decision + * when it later plans the splits — the engine asks this during plan translation, strictly before + * {@link #planScan}. Memoize that decision on the provider instance (the engine keeps one per scan node) + * rather than deciding twice: two independent decisions can disagree, and then BE is asked to read a + * column the tuple does not carry.

+ * + *

Every name returned must be a column of the scanned table, spelled as Doris knows it (the same + * identifier-mapped name {@link #classifyColumn} receives). A name that matches no slot in the scan's + * tuple fails the query loud: it means the connector and the engine disagree about the table, and reading + * on would silently produce whatever the connector's reader does without that column.

+ * + *

The default returns an empty set — every connector whose reader needs nothing beyond the projection + * is untouched, and its scans prune exactly as before.

+ * + * @param session the current session + * @param handle the table handle being scanned + * @return Doris-side names of the columns to read regardless of the projection (default: empty) + */ + default Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + return Collections.emptySet(); + } + /** * Plans the scan described by {@code request}, returning the ranges that cover the requested data. * diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java new file mode 100644 index 00000000000000..f5e6cdb1d8d4dc --- /dev/null +++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.api.scan; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Guards the additive {@code getMustReadColumns} SPI default on {@link ConnectorScanPlanProvider}. + * + *

WHY: the engine consults this on EVERY plugin-table scan that has a projection above it, and widens the + * scan's tuple by whatever comes back. The default must therefore be empty, or every connector that never + * asked for anything would start reading extra columns — and, worse, would fail the query loud when a name + * it never returned matches no slot. This is the zero-break guard for es/jdbc/paimon/iceberg/hive/maxcompute, + * none of which override it.

+ */ +public class ConnectorScanPlanProviderMustReadColumnsTest { + + /** Bare provider: only the abstract planScan implemented; everything else inherits SPI defaults. */ + private static final class BareProvider implements ConnectorScanPlanProvider { + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + return Collections.emptyList(); + } + } + + /** A connector whose BE-side reader needs a merge key the query may not have selected. */ + private static final class KeyReadingProvider implements ConnectorScanPlanProvider { + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + return Collections.emptyList(); + } + + @Override + public Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + return Collections.singleton("id"); + } + } + + @Test + public void defaultAsksForNoExtraColumns() { + ConnectorScanPlanProvider provider = new BareProvider(); + + // MUTATION: a default returning anything non-empty would widen every connector's scans and fail + // loud on the first name that matches no slot -> red here first. + Assertions.assertEquals(Collections.emptySet(), provider.getMustReadColumns(null, null), + "a connector that never opted in must ask for no extra columns"); + } + + @Test + public void connectorThatOptsInIsObeyed() { + ConnectorScanPlanProvider provider = new KeyReadingProvider(); + + Assertions.assertEquals(Collections.singleton("id"), provider.getMustReadColumns(null, null), + "the engine must read back exactly what the connector asked for"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 910aa25e85d610..c5cc0cbaab7e43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -627,6 +627,29 @@ ConnectorColumnCategory classifyColumnByConnector(String columnName) { return onPluginClassLoader(scanProvider, () -> scanProvider.classifyColumn(columnName)); } + /** + * Asks the connector which columns BE must read for this scan even when the query references none of them + * ({@link ConnectorScanPlanProvider#getMustReadColumns}), so the translator can keep their slots instead of + * pruning them ({@code PhysicalPlanTranslator.preserveConnectorMustReadSlots}, the plugin-table counterpart + * of {@code preserveExtraStorageKeySlots} for aggregate / merge-on-read unique-key OLAP tables). + * + *

Asked through the SAME memoized provider the rest of the scan uses, so a connector that memoizes the + * decision on its provider instance answers this question and plans its splits from one decision — the + * whole point, since a column preserved here and a split plan that assumes otherwise disagree silently. + * A connector with no scan provider (no scan capability) needs nothing. Public + overridable because the + * caller is the translator, in another package, and so the preservation is unit-testable without a live + * connector (mirrors {@link #classifyColumnByConnector}, whose caller is this class).

+ */ + public Set mustReadColumnsFromConnector() { + ConnectorScanPlanProvider scanProvider = resolveScanProvider(); + if (scanProvider == null) { + return Collections.emptySet(); + } + Set columns = onPluginClassLoader(scanProvider, + () -> scanProvider.getMustReadColumns(connectorSession, currentHandle)); + return columns == null ? Collections.emptySet() : columns; + } + /** * Lets the owning connector adjust the compression type this node inferred from the split's file path * before it is shipped to BE, WITHOUT any source-specific code here: the base inference runs first, then diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index c09072174ce2d8..60c9a7bfb61914 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -3041,6 +3041,8 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, } if (scanNode instanceof OlapScanNode) { preserveExtraStorageKeySlots((OlapScanNode) scanNode, requiredWithVirtualColumns); + } else if (scanNode instanceof PluginDrivenScanNode) { + preserveConnectorMustReadSlots((PluginDrivenScanNode) scanNode, requiredWithVirtualColumns); } // Find the smallest column, for count(*) or other situation that slot is empty after prune SlotDescriptor smallest = getSmallestSlot(scanNode.getTupleDesc().getSlots()); @@ -3078,6 +3080,43 @@ private void preserveExtraStorageKeySlots(OlapScanNode scanNode, Set req } } + /** + * Keeps the slots of the columns a plugin connector must read for this scan even when the query + * references none of them — the plugin-table counterpart of {@link #preserveExtraStorageKeySlots}, and + * for the same reason: a reader that merges or suppresses rows by key needs the key whether or not the + * user selected it. The connector answers per scan + * ({@code ConnectorScanPlanProvider.getMustReadColumns}, empty by default), so every connector that needs + * nothing beyond the projection prunes exactly as before. + * + *

Only the scan's tuple is widened. The project above it was already given its own output tuple and + * project list a few lines up, so a column preserved here is read and then dropped — it never reaches the + * query's output.

+ * + *

A name that matches no slot fails the query loud rather than being skipped: it means the connector + * and the engine disagree about the table's columns, and the connector's reader would then be handed a + * scan missing a column it said it needs — silently wrong rows, not an error. Static + visible for testing + * so the ask-and-preserve step is pinned without a live connector.

+ */ + @VisibleForTesting + static void preserveConnectorMustReadSlots(PluginDrivenScanNode scanNode, Set requiredSlotIds) { + Set mustRead = scanNode.mustReadColumnsFromConnector(); + if (mustRead.isEmpty()) { + return; + } + Set missing = Sets.newLinkedHashSet(mustRead); + for (SlotDescriptor slot : scanNode.getTupleDesc().getSlots()) { + Column column = slot.getColumn(); + if (column != null && mustRead.contains(column.getName())) { + requiredSlotIds.add(slot.getId()); + missing.remove(column.getName()); + } + } + if (!missing.isEmpty()) { + throw new AnalysisException("connector requires column(s) " + missing + + " to be read, but the scan has no such column"); + } + } + private boolean shouldPreserveStorageKeySlots(OlapScanNode scanNode) { long selectedIndexId = scanNode.getSelectedIndexId() == -1 ? scanNode.getOlapTable().getBaseIndexId() diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java new file mode 100644 index 00000000000000..a5ff8203b9fe73 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.scan; + +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Set; + +/** + * Guards {@link PluginDrivenScanNode#mustReadColumnsFromConnector()} — the seam the translator asks before it + * prunes a plugin table's scan slots ({@code PhysicalPlanTranslator.preserveConnectorMustReadSlots}), so a + * connector whose BE-side reader merges or suppresses rows by key gets that key read even when the query + * never mentions it. + * + *

WHY this matters (Rule 9): the answer is the whole contract between plan translation and split + * planning. Answering for the WRONG handle, or resolving a FRESH provider to ask, would let the connector + * decide "combine the two sources" at split time while the tuple was pruned as if it had said no — and BE + * would be told to suppress rows by a column that is not there. The memo assertion below is what pins + * "asked and planned through one provider instance".

+ * + *

Driven on a {@code CALLS_REAL_METHODS} node with only the connector/session/handle fields injected — + * the same technique as {@link PluginDrivenScanNodeScanProviderSelectionTest}.

+ */ +public class PluginDrivenScanNodeMustReadColumnsTest { + + private static PluginDrivenScanNode nodeWith(ConnectorScanPlanProvider provider, + ConnectorTableHandle handle, ConnectorSession session) { + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Connector connector = Mockito.mock(Connector.class); + Mockito.when(connector.getScanPlanProvider(handle)).thenReturn(provider); + Deencapsulation.setField(node, "connector", connector); + Deencapsulation.setField(node, "currentHandle", handle); + Deencapsulation.setField(node, "connectorSession", session); + return node; + } + + @Test + public void forwardsTheConnectorsAnswerForTheScannedHandle() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(ImmutableSet.of("id", "part")); + PluginDrivenScanNode node = nodeWith(provider, handle, session); + + Set mustRead = node.mustReadColumnsFromConnector(); + + // WHY: the connector answers PER SCAN, from the handle this scan holds. MUTATION: passing a + // different handle (or the table's original one after pushdown refined it) makes the connector + // answer about another read -> the stub returns empty -> red. + Assertions.assertEquals(ImmutableSet.of("id", "part"), mustRead); + Mockito.verify(provider).getMustReadColumns(session, handle); + } + + @Test + public void connectorWithoutScanCapabilityNeedsNothing() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + PluginDrivenScanNode node = nodeWith(null, handle, Mockito.mock(ConnectorSession.class)); + + // WHY: getScanPlanProvider() is null for a connector with no scan capability; every other resolver + // in this node degrades to its default rather than throwing. MUTATION: dropping the null check -> + // NPE during plan translation for such a catalog -> red. + Assertions.assertEquals(Collections.emptySet(), node.mustReadColumnsFromConnector()); + } + + @Test + public void nullAnswerIsReadAsNoExtraColumns() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(null); + PluginDrivenScanNode node = nodeWith(provider, handle, session); + + // WHY: a third-party connector may return null where the SPI says "empty". Turning that into an + // NPE inside plan translation would blame the engine for a connector's slip. MUTATION: returning + // the raw answer -> NPE in the translator's isEmpty() -> red. + Assertions.assertEquals(Collections.emptySet(), node.mustReadColumnsFromConnector()); + } + + @Test + public void asksThroughTheSameProviderInstanceThatWillPlanTheSplits() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(ImmutableSet.of("id")); + Connector connector = Mockito.mock(Connector.class); + Mockito.when(connector.getScanPlanProvider(handle)).thenReturn(provider); + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Deencapsulation.setField(node, "connector", connector); + Deencapsulation.setField(node, "currentHandle", handle); + Deencapsulation.setField(node, "connectorSession", session); + + node.mustReadColumnsFromConnector(); + Object providerForSplits = Deencapsulation.invoke(node, "resolveScanProvider"); + + // WHY: the connector is allowed to memoize "do I combine two sources?" on its provider instance, + // and MUST reach the same answer when it plans the splits later — the columns kept here and the + // splits planned there have to come from one decision. A fresh provider per question loses that + // memo and lets the two disagree. MUTATION: asking via connector.getScanPlanProvider(...) directly + // instead of the memoized resolveScanProvider() -> two instances + a second resolve -> red. + Assertions.assertSame(provider, providerForSplits, + "the must-read question must go through the same memoized provider as split planning"); + Mockito.verify(connector, Mockito.times(1)).getScanPlanProvider(handle); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java new file mode 100644 index 00000000000000..5f1caa9b39d6d2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.scan.PluginDrivenScanNode; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Guards {@code PhysicalPlanTranslator.preserveConnectorMustReadSlots} — the plugin-table branch next to + * {@code preserveExtraStorageKeySlots}, which keeps the slots a connector says its BE-side reader must read + * even when the query references none of them. + * + *

WHY this matters (Rule 9): the scan's tuple is where the columns BE reads are decided + * ({@code FileQueryScanNode.updateRequiredSlots} rebuilds the required-slot list from exactly these slots + * after planning). A reader that suppresses or merges rows by key and does not get the key back reads the + * key-less rows and emits duplicates — no error anywhere. Doris does the same preservation for its own + * aggregate / merge-on-read unique-key tables; this is that mechanism for plugin connectors.

+ * + *

These tests drive the extracted static entry point directly with a real {@link TupleDescriptor} and a + * {@code CALLS_REAL_METHODS} node whose connector answer is stubbed — building a translator over a live + * plugin catalog needs a harness this module does not have. What they do NOT cover, because it is decided + * before this branch runs and by generic code: the project above the scan gets its own output tuple, so a + * column preserved here is read and then dropped rather than returned. The fluss suites' row baselines are + * the end-to-end guard for that.

+ */ +public class PhysicalPlanTranslatorMustReadSlotsTest { + + private static final DescriptorTable DESC_TABLE = new DescriptorTable(); + + /** A scan tuple holding one slot per named column, in order. */ + private static TupleDescriptor tupleOf(String... columnNames) { + TupleDescriptor tuple = DESC_TABLE.createTupleDescriptor(); + for (String name : columnNames) { + SlotDescriptor slot = DESC_TABLE.addSlotDescriptor(tuple); + slot.setColumn(new Column(name, PrimitiveType.INT)); + } + return tuple; + } + + private static PluginDrivenScanNode nodeAnswering(TupleDescriptor tuple, Set mustRead) { + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(tuple).when(node).getTupleDesc(); + Mockito.doReturn(mustRead).when(node).mustReadColumnsFromConnector(); + return node; + } + + private static SlotId slotIdOf(TupleDescriptor tuple, String columnName) { + for (SlotDescriptor slot : tuple.getSlots()) { + if (slot.getColumn().getName().equals(columnName)) { + return slot.getId(); + } + } + throw new IllegalStateException("no slot for " + columnName); + } + + @Test + public void connectorNamedColumnsSurvivePruning() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id")); + // "select name": only that slot is required by the project above the scan. + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: 'id' is what the connector's reader needs to suppress rows; without it in the required set + // the removeIf below this call drops it from the tuple and BE reads key-less rows. MUTATION: + // dropping the branch (or the add) -> 'id' absent -> red. + Assertions.assertEquals(ImmutableSet.of(slotIdOf(tuple, "name"), slotIdOf(tuple, "id")), required); + } + + @Test + public void connectorThatNeedsNothingChangesNothing() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, Collections.emptySet()); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: this is the gate that keeps the branch inert for every connector that never opted in — and + // for an opted-in connector on a scan it decided NOT to combine (a fluss table read from fluss + // alone), which is exactly the "only when it is really needed" requirement. MUTATION: preserving + // unconditionally (e.g. the whole primary key regardless of the decision) -> an extra slot -> red. + Assertions.assertEquals(Collections.singleton(slotIdOf(tuple, "name")), required); + } + + @Test + public void everyNamedColumnIsPreservedNotJustTheFirst() { + TupleDescriptor tuple = tupleOf("k1", "k2", "payload"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("k1", "k2")); + Set required = Sets.newHashSet(slotIdOf(tuple, "payload")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: composite keys are the normal case for the readers this exists for; keeping only one column + // of a two-column key compares the wrong thing. MUTATION: `break` after the first match -> red. + Assertions.assertEquals( + ImmutableSet.of(slotIdOf(tuple, "payload"), slotIdOf(tuple, "k1"), slotIdOf(tuple, "k2")), + required); + } + + @Test + public void preservedSlotSurvivesThePruneItself() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id")); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + // The real prune step, driven end to end: it is what decides which slots the scan reads, and the + // branch under test sits inside it. + Deencapsulation.invoke(new PhysicalPlanTranslator(), "updateScanSlotsMaterialization", + node, required, Sets.newHashSet(), new PlanTranslatorContext()); + + // WHY: this is the only assertion that also pins the DISPATCH — that a plugin-driven scan reaches + // the branch at all. MUTATION: deleting the `else if (scanNode instanceof PluginDrivenScanNode)` + // arm -> 'id' pruned away -> red. MUTATION: preserving AFTER the removeIf -> also red. + Assertions.assertEquals(ImmutableList.of("id", "name"), + tuple.getSlots().stream().map(s -> s.getColumn().getName()).collect(Collectors.toList()), + "the connector's column must be read; the unreferenced one must still be pruned"); + } + + @Test + public void columnTheScanDoesNotHaveFailsLoud() { + TupleDescriptor tuple = tupleOf("id", "name"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id", "ghost")); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + AnalysisException thrown = Assertions.assertThrows(AnalysisException.class, + () -> PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required)); + + // WHY: a name matching no slot means the connector and the engine disagree about the table. Reading + // on would hand the reader a scan without a column it said it needs — wrong rows, silently. The + // message must name the column, because that is the only clue to which side is stale. MUTATION: + // skipping unknown names instead of throwing -> no exception -> red. + Assertions.assertTrue(thrown.getMessage().contains("ghost"), + "the failure must name the column the scan does not have: " + thrown.getMessage()); + } +} From 46a6dc7549edb250767bb1d418dc2d6df2e27818 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 20:26:55 +0800 Subject: [PATCH 29/35] [feat](fluss) Plan a primary-key lake table as its lake plus its log tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tiered primary-key table was read from fluss alone: correct, because fluss keeps such a table's state in full, but it left the lake's columnar files unread. It is now planned as three parts, per bucket: - the sibling's lake splits, wrapped with the offsets of the log tail that supersedes part of them (BE drops the lake rows whose keys the tail names), - one PK_TAIL range producing the tail's own surviving state, exactly once, - PK_FULL for a bucket the lake has never seen, as before. A split is bound to the tail of its OWN bucket, using the bucket the paimon connector now reports on each range. Three ways that can go wrong all fail loud, because each would silently duplicate rows: a split that names no bucket (an older paimon plugin), a bucket this table does not have (the two are not bucketed alike), a bucket the lake holds files for but fluss records no tiering offset for (their metadata disagrees). Two conditions make the union unsafe rather than merely awkward, and under auto both give up the lake half instead of failing — the fluss-only read they fall back to is the whole table: - a key or partition column whose values may not compare or render the same way on both sides (float, a timestamp Doris rounds, a non-string partition column, all of which fluss itself allows). Settled from the schema alone, before anything is asked of the lake, so that the answer is the same at plan translation time as it is when the ranges are planned; - a tail the log no longer holds. Fluss deletes old log segments on a timer that does not wait for tiering, and a primary-key table cannot re-read its log from the lake, so planning verifies the tail is still there. required refuses both, as it must: that mode exists so a test can assert the lake was actually read. EXPLAIN gains suppressedLakeSplits, pkTailRanges and a degraded= reason, which is otherwise the only thing distinguishing "there is no lake" from "there is one this query could not use". The scan keeps the key columns in its tuple through the SPI added for it, so BE reads them as ordinary projected columns; the projection above the scan drops them again. Both questions are answered from one memoized resolution, which is a correctness requirement rather than a saving. Removes the UNION_PK range that an earlier design would have used. 182 tests, 0 skipped. Ten mutations were confirmed to fail the new assertions, including binding splits by bucket while ignoring the partition, reading a missing bucket property as "no tail", skipping the tail guard, and resolving the union read twice instead of reusing the memo. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussConnectorProperties.java | 44 +- .../fluss/FlussScanPlanProvider.java | 525 ++++++++++++-- .../doris/connector/fluss/FlussScanRange.java | 61 +- .../fluss/FlussSuppressedLakeRange.java | 232 +++++++ .../connector/fluss/FlussTableHandle.java | 61 +- .../connector/fluss/FlussUnionKeyTypes.java | 111 +++ .../fluss/FlussConnectorPropertiesTest.java | 26 + .../connector/fluss/FlussScanRangeTest.java | 58 +- .../connector/fluss/FlussSplitPlanTest.java | 638 +++++++++++++++++- .../fluss/FlussSuppressedLakeRangeTest.java | 217 ++++++ .../connector/fluss/FlussTableHandleTest.java | 22 +- .../fluss/FlussUnionKeyTypesTest.java | 135 ++++ .../fluss/RecordingFlussAdminOps.java | 15 +- .../connector/fluss/RecordingLakeSibling.java | 43 +- gensrc/thrift/PlanNodes.thrift | 12 +- 15 files changed, 2045 insertions(+), 155 deletions(-) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java index 12df540e4de99f..954903ac476443 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java @@ -57,6 +57,23 @@ public final class FlussConnectorProperties { */ public static final String UNION_READ_MODE = "fluss.union_read.mode"; + /** + * Optional. How many log-tail rows one bucket of a primary-key table may contribute to a union read. + * + *

Doris-only. Reading a primary-key table's lake together with its log tail holds that tail in BE + * memory — once as the set of keys that suppress lake rows, once as the rows the tail itself + * contributes — so an unbounded tail is an unbounded allocation inside a process that serves every + * other query too. Fluss's own engines have no such limit (Flink's tiering reader accumulates the + * tail on the heap and lets the task manager die and restart if it does not fit), which is exactly + * the difference: BE has no restart to fall back on. A tail is by design the data written within one + * {@code table.datalake.freshness} period, so the default is far above any healthy table; hitting it + * means tiering has stalled, and the error says so. + */ + public static final String UNION_READ_MAX_TAIL_ROWS = "fluss.union_read.max_tail_rows"; + + /** Default {@link #UNION_READ_MAX_TAIL_ROWS}: two million rows per bucket. */ + public static final long DEFAULT_MAX_TAIL_ROWS = 2_000_000L; + /** * Optional. Whether a fluss BINARY/BYTES column reads as Doris VARBINARY instead of STRING. * @@ -109,6 +126,7 @@ private FlussConnectorProperties() { public static void validate(Map properties) { validateBootstrapServers(bootstrapServers(properties)); unionReadMode(properties); + maxTailRows(properties); } /** The declared bootstrap servers, or the empty string when the property is absent. */ @@ -123,6 +141,30 @@ public static UnionReadMode unionReadMode(Map properties) { return value == null ? UnionReadMode.AUTO : UnionReadMode.parse(value); } + /** + * The declared per-bucket tail ceiling, {@link #DEFAULT_MAX_TAIL_ROWS} when the property is absent. + * Zero and negative values are rejected rather than read as "no limit": a user who wants no ceiling + * has no way to say so, which is the point. + */ + public static long maxTailRows(Map properties) { + String value = properties.get(UNION_READ_MAX_TAIL_ROWS); + if (value == null) { + return DEFAULT_MAX_TAIL_ROWS; + } + long rows; + try { + rows = Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + UNION_READ_MAX_TAIL_ROWS + "'; expected a positive number of rows"); + } + if (rows <= 0) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + UNION_READ_MAX_TAIL_ROWS + "'; expected a positive number of rows"); + } + return rows; + } + /** The type-mapping switches this catalog declares; both default to off. */ public static FlussTypeMapping.Options typeMappingOptions(Map properties) { return new FlussTypeMapping.Options( @@ -155,7 +197,7 @@ public static Map toFlussClientConfig(Map proper /** Whether {@code key} configures Doris's own behaviour and must not reach the fluss client. */ private static boolean isDorisOnly(String key) { - return UNION_READ_MODE.equals(key); + return UNION_READ_MODE.equals(key) || UNION_READ_MAX_TAIL_ROWS.equals(key); } private static void validateBootstrapServers(String value) { diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java index 984420c454a7f9..e343547ce11b58 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -39,11 +39,13 @@ import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -78,12 +80,23 @@ * as fluss-only silently returns just the rows tiering has not moved yet, which looks like a working * query. {@code fluss.union_read.mode=disabled} is how a user asks for that fluss-only read on purpose. * - *

A tiered PRIMARY-KEY table is read from fluss alone for now. Its two halves cannot be concatenated — - * the log carries updates and deletes of rows the lake already holds, so they have to be merged BY KEY — - * and that merge is not implemented. Falling back is safe here in a way it is not for a log table: fluss - * keeps a primary-key table's state in full, so the fluss-only read is the whole table, only slower than - * reading the lake's columnar files would be. {@code required} refuses such a table anyway, because that - * mode exists to make "did this actually read the lake?" answerable in a test. + *

A tiered PRIMARY-KEY table is read as a union too, but its halves overlap by KEY rather than meeting + * at an offset: the log tail carries updates and deletes of rows the lake already holds. It is split into + * three parts, per bucket. The lake splits of a bucket whose log has moved on are wrapped + * ({@link FlussSuppressedLakeRange}) with the offsets of that tail, and BE drops the lake rows whose keys + * the tail names. The surviving state of the tail itself is contributed once, by a + * {@code PK_TAIL} range. A bucket the lake has never seen is read whole from fluss, as {@code PK_FULL}, + * exactly as it would be without a lake. Every row is therefore produced exactly once, and the read + * matches what fluss alone would return — which is what makes the fluss-only read the reference the + * regression tests compare against. + * + *

Falling back to that fluss-only read is always safe for a primary-key table, in a way it is not for + * a log table: fluss keeps such a table's state in full, so reading it alone returns the whole table, + * only slower than reading the lake's columnar files would be. That is why {@code auto} answers every + * question it cannot answer well — a key column it cannot compare exactly, a tail the log no longer + * holds — by reading fluss alone rather than by failing, and why {@code required} answers the same + * questions with an error: that mode exists to make "did this actually read the lake?" answerable in a + * test. */ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { @@ -103,6 +116,26 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { static final String PROP_DB_NAME = "fluss.db_name"; static final String PROP_TABLE_NAME = "fluss.table_name"; + /** + * Node property naming the key columns BE compares when it suppresses lake rows: the primary key + * minus the partition columns, in Doris's own column names. Only the names — the types travel the + * ordinary way, as the slot descriptors of columns that {@link #getMustReadColumns} kept in the + * scan's tuple. + */ + static final String PROP_UNION_PK_NAMES = "fluss.union.pk_names"; + + /** Node property carrying {@link FlussConnectorProperties#UNION_READ_MAX_TAIL_ROWS} to both readers. */ + static final String PROP_UNION_MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; + + /** + * The property a lake split carries to say which bucket it holds. Written by the paimon connector + * (from {@code DataSplit.bucket()}) for this connector's sake, and part of the contract between the + * two: a fluss table's lake table is bucketed identically, so a lake split can be matched with the + * log tail of the SAME bucket. Its absence is never treated as "this bucket has no tail" — that would + * turn a version mismatch into duplicated rows — see {@link #lakeSplitBucket}. + */ + private static final String LAKE_BUCKET_PROPERTY = "paimon.bucket"; + /** The only lake format that can be delegated today; fluss also defines iceberg / lance / hudi. */ private static final String PAIMON_LAKE_FORMAT = "paimon"; @@ -119,7 +152,9 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { */ private int plannedLogRanges; private int plannedPkRanges; + private int plannedPkTailRanges; private int plannedLakeSplits; + private int plannedSuppressedLakeSplits; private boolean plannedUnionRead; /** @@ -130,6 +165,24 @@ public class FlussScanPlanProvider implements ConnectorScanPlanProvider { private boolean unionResolved; private UnionRead unionRead; + /** + * Why this scan gave up its lake half, or null when it did not. Only {@code auto} can get here — the + * same conditions are errors under {@code required} — and the plan that results is the fluss-only read + * {@code disabled} would have produced, which for a primary-key table is the whole table. It shows up + * in EXPLAIN because that is otherwise the only difference between "there is no lake to read" and + * "there is one and this query could not use it". + */ + private String degradedReason; + + /** {@link #degradedReason} when the log no longer holds the tail the lake snapshot stops before. */ + private static final String DEGRADED_TAIL_TRUNCATED = "tail-truncated"; + + /** {@link #degradedReason} when a key column's values cannot be compared exactly across the halves. */ + private static final String DEGRADED_KEY_TYPE = "key-type"; + + /** {@link #degradedReason} when a partition column's values do not render the same way on both sides. */ + private static final String DEGRADED_PARTITION_TYPE = "partition-type"; + public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties, Function, Connector> lakeSiblingFactory) { this.adminOps = adminOps; @@ -165,15 +218,35 @@ public List planScan(ConnectorSession session, ConnectorScan UnionRead union = resolveUnionRead(session, handle); List buckets = allBuckets(handle.getBucketCount()); - List ranges = new ArrayList<>(); + List ranges = union != null && handle.hasPrimaryKey() + ? planPrimaryKeyUnion(session, handle, union, buckets, request) + : planWithoutKeyMerging(session, handle, union, buckets, request); + + plannedLogRanges = count(ranges, FlussScanRange.RangeType.LOG); + plannedPkRanges = count(ranges, FlussScanRange.RangeType.PK_FULL); + plannedPkTailRanges = count(ranges, FlussScanRange.RangeType.PK_TAIL); + plannedLakeSplits = countLakeSplits(ranges); + plannedSuppressedLakeSplits = countSuppressedLakeSplits(ranges); + // Read back from the field, not from the local: a primary-key plan may have given up its lake half + // half-way through, and EXPLAIN has to say what was actually planned. + plannedUnionRead = unionRead != null; + return ranges; + } + /** + * A log table, or any table read from fluss alone: every bucket end to end, with the lake half — when + * there is one — prepended. + */ + private List planWithoutKeyMerging(ConnectorSession session, + FlussTableHandle handle, UnionRead union, List buckets, + ConnectorScanRequest request) { + List ranges = new ArrayList<>(); // The lake half first, so its ranges lead the list the way they lead the table's history. It is // planned once for the whole table: the sibling prunes partitions from the pushed-down filter, not // from the engine's pruned partition list (which it does not consume). if (union != null) { ranges.addAll(planLakeRanges(session, union, request)); } - plannedLakeSplits = ranges.size(); if (handle.isPartitioned()) { for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { @@ -185,13 +258,270 @@ public List planScan(ConnectorSession session, ConnectorScan } else { appendPartitionRanges(ranges, handle, union, FlussScanRange.Partition.NONE, buckets, null); } + return ranges; + } - plannedLogRanges = count(ranges, FlussScanRange.RangeType.LOG); - plannedPkRanges = count(ranges, FlussScanRange.RangeType.PK_FULL); - plannedUnionRead = union != null; + /** + * A primary-key table read as its lake plus the log written since: lake splits bound to the tail of + * their own bucket, plus one range per bucket for what the lake does not hold. + * + *

The offsets are read BEFORE the lake half is planned, and that ordering carries the last guard of + * the design (D17): a tail the log no longer holds cannot be read at all, and a primary-key table + * cannot fetch it from the lake instead. Finding that out after asking the sibling to plan would mean + * throwing its work away; finding it out here means the fluss-only read that replaces it is planned + * from the very same offsets, and is the plan {@code disabled} would have produced. + */ + private List planPrimaryKeyUnion(ConnectorSession session, + FlussTableHandle handle, UnionRead union, List buckets, + ConnectorScanRequest request) { + List states = new ArrayList<>(); + if (handle.isPartitioned()) { + for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { + states.add(readPartitionState(handle, union, + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), + buckets, partition.getPartitionName())); + } + } else { + states.add(readPartitionState(handle, union, FlussScanRange.Partition.NONE, buckets, null)); + } + + String truncated = firstTruncatedTail(states, buckets); + if (truncated != null) { + if (FlussConnectorProperties.unionReadMode(catalogProperties) + == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read as its lake plus its log: " + truncated + + ". Fluss has already deleted part of the log the lake snapshot stops before, and a" + + " primary-key table's log cannot be re-read from the lake. Set '" + + FlussConnectorProperties.UNION_READ_MODE + "' to auto or disabled to read the" + + " table from fluss alone, which still returns every row."); + } + degradeToFlussOnly(DEGRADED_TAIL_TRUNCATED); + return pkRangesFromFlussAlone(states, buckets); + } + + List ranges = new ArrayList<>(); + for (ConnectorScanRange lakeSplit : planLakeRanges(session, union, request)) { + ranges.add(bindTailToLakeSplit(handle, lakeSplit, states)); + } + for (PartitionState state : states) { + for (int bucket : buckets) { + BucketState bucketState = state.buckets.get(bucket); + if (bucketState.lakeEnd == null) { + // Never tiered: the lake holds nothing of this bucket, so it is read exactly as it + // would be with no lake at all, and no lake split of it can be suppressed. + appendPkFullRange(ranges, state, bucket); + } else if (bucketState.lakeEnd < bucketState.stop) { + ranges.add(FlussScanRange.pkTail(state.partition, bucket, + bucketState.lakeEnd, bucketState.stop)); + } + // lakeEnd == stop: the lake holds this bucket entirely and fluss adds nothing. + } + } return ranges; } + /** What planning read about one partition of a primary-key table, or about an unpartitioned one. */ + private static final class PartitionState { + private final FlussScanRange.Partition partition; + private final KvSnapshots snapshots; + private final Map buckets; + + private PartitionState(FlussScanRange.Partition partition, KvSnapshots snapshots, + Map buckets) { + this.partition = partition; + this.snapshots = snapshots; + this.buckets = buckets; + } + } + + /** Where one bucket's log begins, ends, and how much of it the lake already holds. */ + private static final class BucketState { + /** First offset NOT in the lake, or null when the lake snapshot does not mention this bucket. */ + private final Long lakeEnd; + /** Where planning saw the log end; 0 for a bucket nothing has ever been written to. */ + private final long stop; + /** The earliest offset fluss still holds, or null when it did not answer for this bucket. */ + private final Long earliest; + + private BucketState(Long lakeEnd, long stop, Long earliest) { + this.lakeEnd = lakeEnd; + this.stop = stop; + this.earliest = earliest; + } + } + + /** + * Everything planning needs about one partition, read in the one order that is safe: kv snapshots, + * then the offsets that bound them, then the earliest offsets the guard compares against. + * + *

The kv snapshots are read even when every bucket turns out to be tiered and none of them is used. + * That is deliberate: giving up the lake half is decided AFTER the offsets are in hand, and the + * fluss-only plan that replaces it needs the snapshots — read at that point they would be newer than + * the offsets that bound them, which is the one ordering this planner exists to prevent. + */ + private PartitionState readPartitionState(FlussTableHandle handle, UnionRead union, + FlussScanRange.Partition partition, List buckets, String flussPartitionName) { + TablePath tablePath = handle.toTablePath(); + KvSnapshots snapshots = latestKvSnapshots(tablePath, flussPartitionName); + Map stopping = latestOffsets(tablePath, flussPartitionName, buckets); + // Last, and that order is what makes it conservative: the earliest offset only moves forward, so + // one read later can only make the guard stricter, never let a truncated tail through. + Map earliest = earliestOffsets(tablePath, flussPartitionName, buckets); + + Map byBucket = new LinkedHashMap<>(); + for (int bucket : buckets) { + TableBucket tableBucket = partition.isPartitioned() + ? new TableBucket(handle.getTableId(), partition.getId(), bucket) + : new TableBucket(handle.getTableId(), bucket); + Long stop = stopping.get(bucket); + byBucket.put(bucket, new BucketState(union.logOffsets.get(tableBucket), + stop == null ? 0L : stop, earliest.get(bucket))); + } + return new PartitionState(partition, snapshots, byBucket); + } + + /** + * The first bucket whose tail fluss can no longer serve, described for an error message, or null when + * every tail is intact. + * + *

A tail is the log from where the lake snapshot ended up to where planning saw the log. Fluss + * deletes old log segments on a timer that does not wait for tiering (only for the kv snapshot), so on + * a cluster without remote log storage the beginning of that tail can be gone. Reading it anyway + * returns fewer rows than the table holds, and unlike a log table there is nowhere else to get them + * from — the lake's copy stops exactly where the missing tail begins. + * + *

A bucket fluss did not answer for at all counts as truncated: the check is the only thing + * standing between a deleted tail and a silently short answer, and "could not verify" is not "fine". + */ + private static String firstTruncatedTail(List states, List buckets) { + for (PartitionState state : states) { + for (int bucket : buckets) { + BucketState bucketState = state.buckets.get(bucket); + if (bucketState.lakeEnd == null || bucketState.lakeEnd >= bucketState.stop) { + continue; + } + String where = (state.partition.isPartitioned() + ? "partition '" + state.partition.getName() + "', " : "") + "bucket " + bucket; + if (bucketState.earliest == null) { + return where + " (fluss did not report an earliest offset for it)"; + } + if (bucketState.earliest > bucketState.lakeEnd) { + return where + " (the lake ends at offset " + bucketState.lakeEnd + + ", but the log now starts at " + bucketState.earliest + ")"; + } + } + } + return null; + } + + /** Every bucket read whole from fluss: the plan {@code disabled} produces, and the safe fallback. */ + private static List pkRangesFromFlussAlone(List states, + List buckets) { + List ranges = new ArrayList<>(); + for (PartitionState state : states) { + for (int bucket : buckets) { + appendPkFullRange(ranges, state, bucket); + } + } + return ranges; + } + + /** + * Gives up this scan's lake half for {@code reason}. Everything asked afterwards — the node + * properties, the scan-level params, EXPLAIN — then answers as a fluss-only read, because the field + * the answers come from is the one being cleared here. + * + *

One question was asked earlier and cannot be taken back: {@link #getMustReadColumns}, at plan + * translation time, may already have kept the key columns in the scan's tuple. That is harmless in + * this direction — they are read and then dropped by the projection above the scan — and it is the + * reason this guard is allowed to run so late. The opposite order would not be harmless, which is why + * the conditions that CAN be decided before translation (a key column's type) are decided there. + */ + private void degradeToFlussOnly(String reason) { + degradedReason = reason; + unionRead = null; + } + + /** + * The lake split, wrapped with the log tail of its own bucket when that tail holds anything. + * + *

Which bucket a split belongs to is the sibling's answer ({@code paimon.bucket}); which partition + * it belongs to is compared by the rendered partition values, which is sound because a union read of a + * partitioned table is refused unless the partition columns are strings. Three things can go wrong, + * and none of them may be waved through: a split with no bucket at all means the paimon connector does + * not write the property this contract is built on; a bucket number this table does not have means the + * two tables are not bucketed alike; a bucket the lake holds files for but fluss records no tiering + * offset for means their metadata disagrees. Each would silently duplicate rows. + */ + private ConnectorScanRange bindTailToLakeSplit(FlussTableHandle handle, ConnectorScanRange lakeSplit, + List states) { + PartitionState state = matchingPartition(lakeSplit, states); + if (state == null) { + // A partition of the lake that this scan does not read from fluss: either one fluss has since + // dropped, or one the engine pruned away. Nothing of it can be superseded by a tail this plan + // does not read, so the split is passed through untouched. + return lakeSplit; + } + int bucket = lakeSplitBucket(handle, lakeSplit); + BucketState bucketState = state.buckets.get(bucket); + if (bucketState == null) { + throw new DorisConnectorException("The lake table of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' has a split in bucket " + bucket + ", but the fluss" + + " table has only " + handle.getBucketCount() + " buckets. The two are not bucketed" + + " alike, so their rows cannot be matched by bucket"); + } + if (bucketState.lakeEnd == null) { + throw new DorisConnectorException("The lake table of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' holds data for bucket " + bucket + + (state.partition.isPartitioned() + ? " of partition '" + state.partition.getName() + "'" : "") + + ", but fluss records no tiering offset for that bucket. Their metadata disagrees;" + + " reading them as one would return the rows of that bucket twice"); + } + if (bucketState.lakeEnd >= bucketState.stop) { + // The lake holds this bucket up to where its log ends: nothing can supersede it. + return lakeSplit; + } + return new FlussSuppressedLakeRange(lakeSplit, new FlussSuppressedLakeRange.Tail( + state.partition, bucket, bucketState.lakeEnd, bucketState.stop)); + } + + /** + * The partition this scan planned that {@code lakeSplit} belongs to, or null when it planned none. + * Matched on the partition values as a whole rather than on a rendered name, so that there is exactly + * one place in this connector that turns partition values into a name (see {@link FlussPartitions}). + */ + private static PartitionState matchingPartition(ConnectorScanRange lakeSplit, + List states) { + Map values = lakeSplit.getPartitionValues(); + for (PartitionState state : states) { + if (state.partition.getValues().equals(values)) { + return state; + } + } + return null; + } + + /** The bucket a lake split holds, per the contract in {@link #LAKE_BUCKET_PROPERTY}. */ + private static int lakeSplitBucket(FlussTableHandle handle, ConnectorScanRange lakeSplit) { + String bucket = lakeSplit.getProperties().get(LAKE_BUCKET_PROPERTY); + if (bucket == null) { + throw new DorisConnectorException("A lake split of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' does not say which bucket it holds ('" + + LAKE_BUCKET_PROPERTY + "' is missing). Reading a primary-key table together with its" + + " lake needs it, to bind each split to the log tail of the same bucket; the paimon" + + " connector plugin is older than this fluss connector plugin. Split: " + lakeSplit); + } + try { + return Integer.parseInt(bucket); + } catch (NumberFormatException e) { + throw new DorisConnectorException("A lake split of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' reports bucket '" + bucket + "', which is not a" + + " number", e); + } + } + /** * The lake half, planned by the sibling connector on the handle already pinned to the readable snapshot. * @@ -271,27 +601,27 @@ private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableH return null; } if (handle.hasPrimaryKey()) { - // A primary-key table's lake is NOT read, and the fluss-only read that replaces it is the whole - // table rather than a part of it: fluss keeps the table's state in its own kv store, and tiering - // copies rows into the lake without taking them out of it, so the latest kv snapshot plus the - // change log after it is every row. That is the opposite of a log table, where whatever tiering - // has aged out of the log exists only in the lake and a fluss-only read silently loses it — which - // is why that case falls back only when the lake holds nothing yet. What is lost here is speed, - // not rows: the bucket's whole kv snapshot is fetched and merged instead of reading the lake's - // columnar files. Merging the two by key is a separate piece of work. - // - // Asked before the lake snapshot is, deliberately: the answer cannot depend on it. A table whose - // tiering has not committed yet would otherwise report the wrong reason under 'required', and - // waiting for that commit would not help. - if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { - throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." - + handle.getTableName() + "' is a primary-key table tiered into a lake, and '" - + FlussConnectorProperties.UNION_READ_MODE + "=required' asks for its lake and its" - + " change log to be read as one. Merging them by key is not implemented yet. Reading" - + " it from fluss alone still returns the whole table, so set the property to auto or" - + " disabled."); + // Whether the two halves can be matched by key at all is decided HERE, before the lake snapshot + // is asked for, because it depends only on the table's schema — and because this is the one + // decision that must be the same at plan-translation time (when getMustReadColumns keeps the key + // columns in the scan's tuple) as it is later, when the ranges are planned. Everything that can + // only be known from the offsets is decided in planScan, where giving up is still safe. + String rejection = keyColumnRejection(handle); + String reason = DEGRADED_KEY_TYPE; + if (rejection == null) { + rejection = partitionColumnRejection(handle); + reason = DEGRADED_PARTITION_TYPE; + } + if (rejection != null) { + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read as its lake plus its change log: " + + rejection + ". Reading it from fluss alone still returns every row, so set '" + + FlussConnectorProperties.UNION_READ_MODE + "' to auto or disabled."); + } + degradedReason = reason; + return null; } - return null; } LakeSnapshot snapshot; try { @@ -341,6 +671,62 @@ private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableH snapshot.getTableBucketsOffset()); } + /** + * Why this table's rows cannot be matched across the two halves by their keys, or null when they can. + * The key is the primary key minus the partition columns, because a bucket lives inside one partition + * and the partition columns are equal for every row in it. + */ + private static String keyColumnRejection(FlussTableHandle handle) { + for (String column : handle.getPhysicalPrimaryKeys()) { + DataType type = handle.getKeyColumnTypes().get(column); + if (type == null) { + // The handle names a key column the schema it was built from does not have. + return "its primary key names column '" + column + "', which the table does not have"; + } + String rejection = FlussUnionKeyTypes.keyColumnRejection(type); + if (rejection != null) { + return "primary-key column '" + column + "' has type " + type + ", and " + rejection; + } + } + return null; + } + + /** The same question for the partition columns, which decide which bucket a lake split belongs to. */ + private static String partitionColumnRejection(FlussTableHandle handle) { + for (String column : handle.getPartitionKeys()) { + DataType type = handle.getKeyColumnTypes().get(column); + if (type == null) { + return "it is partitioned by column '" + column + "', which the table does not have"; + } + String rejection = FlussUnionKeyTypes.partitionColumnRejection(type); + if (rejection != null) { + return "partition column '" + column + "' has type " + type + ", and " + rejection; + } + } + return null; + } + + /** + * The key columns BE has to read whether or not the query asked for them, so that it can tell which + * lake rows the log tail supersedes. Empty for every other read, which is every read but this one. + * + *

Asked by the engine while it is translating the plan, well before {@link #planScan}, and answered + * from the SAME memoized resolution — that is a correctness requirement, not a saving. If the two + * answers could differ, a scan whose tuple was pruned as a fluss-only read could still be planned as a + * union read, and BE would look for a key column that is not in its projection. + * + *

Only a primary-key table is asked further, so that a log table's scan does not pull the lake + * snapshot's round trip forward into plan translation for nothing. + */ + @Override + public Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + if (!flussHandle.hasPrimaryKey() || resolveUnionRead(session, flussHandle) == null) { + return Collections.emptySet(); + } + return new LinkedHashSet<>(flussHandle.getPhysicalPrimaryKeys()); + } + /** * The ranges covering one partition of a table, or the whole of an unpartitioned one * ({@code flussPartitionName} is null, which is also how the two admin overloads are told apart). @@ -381,6 +767,15 @@ private Map latestOffsets(TablePath tablePath, String flussPartit : adminOps.listOffsets(tablePath, flussPartitionName, buckets, new OffsetSpec.LatestSpec()); } + /** The earliest offset each bucket's log still holds — what a tail has to still begin at or after. */ + private Map earliestOffsets(TablePath tablePath, String flussPartitionName, + List buckets) { + return flussPartitionName == null + ? adminOps.listOffsets(tablePath, buckets, new OffsetSpec.EarliestSpec()) + : adminOps.listOffsets(tablePath, flussPartitionName, buckets, + new OffsetSpec.EarliestSpec()); + } + /** * The partitions to scan: those the engine's pruning left, or all of them when it pruned nothing. * A pruned name that fluss no longer lists is simply absent from the result — the partition was @@ -458,19 +853,29 @@ private static void appendPkRanges(List ranges, FlussScanRange.Partition partition, List buckets, KvSnapshots snapshots, Map stopping) { for (int bucket : buckets) { - long snapshotId = snapshots.getSnapshotId(bucket).orElse(FlussScanRange.NO_KV_SNAPSHOT); - long logStart = snapshots.getLogOffset(bucket).orElse(LogScanner.EARLIEST_OFFSET); Long stop = stopping.get(bucket); - long logStop = stop == null ? 0L : stop; - if (snapshotId == FlussScanRange.NO_KV_SNAPSHOT && logStop <= 0) { - // Nothing snapshotted and nothing logged: the bucket is empty. A bucket WITH a snapshot - // is planned even when its log has caught up, because the snapshot still holds rows. - continue; - } - ranges.add(FlussScanRange.pkFull(partition, bucket, snapshotId, logStart, logStop)); + appendPkFullRange(ranges, partition, bucket, snapshots, stop == null ? 0L : stop); } } + /** The same, for a bucket of a table whose lake half is being read too. */ + private static void appendPkFullRange(List ranges, PartitionState state, + int bucket) { + appendPkFullRange(ranges, state.partition, bucket, state.snapshots, state.buckets.get(bucket).stop); + } + + private static void appendPkFullRange(List ranges, + FlussScanRange.Partition partition, int bucket, KvSnapshots snapshots, long logStop) { + long snapshotId = snapshots.getSnapshotId(bucket).orElse(FlussScanRange.NO_KV_SNAPSHOT); + long logStart = snapshots.getLogOffset(bucket).orElse(LogScanner.EARLIEST_OFFSET); + if (snapshotId == FlussScanRange.NO_KV_SNAPSHOT && logStop <= 0) { + // Nothing snapshotted and nothing logged: the bucket is empty. A bucket WITH a snapshot + // is planned even when its log has caught up, because the snapshot still holds rows. + return; + } + ranges.add(FlussScanRange.pkFull(partition, bucket, snapshotId, logStart, logStop)); + } + /** * How many of {@code ranges} are fluss ranges of this kind. The type test is not defensive: on a union * read the list also holds the lake half's ranges, which are the sibling's own type and would fail a @@ -486,6 +891,27 @@ private static int count(List ranges, FlussScanRange.RangeTy return found; } + /** Ranges the sibling planned, counted by exclusion: a wrapped one is still one of the lake's. */ + private static int countLakeSplits(List ranges) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (!(range instanceof FlussScanRange)) { + found++; + } + } + return found; + } + + private static int countSuppressedLakeSplits(List ranges) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (range instanceof FlussSuppressedLakeRange) { + found++; + } + } + return found; + } + private static List allBuckets(int bucketCount) { List buckets = new ArrayList<>(bucketCount); for (int bucket = 0; bucket < bucketCount; bucket++) { @@ -516,6 +942,14 @@ public Map getScanNodeProperties(ConnectorSession session, Conne UnionRead union = resolveUnionRead(session, flussHandle); if (union != null) { + if (flussHandle.hasPrimaryKey()) { + // What BE needs to suppress lake rows by key: which columns the key is made of, and how + // large a tail it may hold in memory while doing so. Both are node-level because both are + // the same for every range of the scan. + props.put(PROP_UNION_PK_NAMES, String.join(",", flussHandle.getPhysicalPrimaryKeys())); + props.put(PROP_UNION_MAX_TAIL_ROWS, + String.valueOf(FlussConnectorProperties.maxTailRows(catalogProperties))); + } List lakeColumns = lakeColumns(session, union, columns, handle); mergeLakeProperties(props, LakeSibling.call(union.sibling, () -> union.siblingProvider.getScanNodeProperties( @@ -601,10 +1035,17 @@ public void appendExplainInfo(StringBuilder output, String prefix, MapThese are the kinds the java scanner reads. A union read of a primary-key table puts one more + * value on the wire — the suppression descriptor that rides along with a LAKE split, see + * {@link FlussSuppressedLakeRange} — which never reaches the scanner: BE's C++ side reads it to + * build the key set that hides superseded lake rows. + */ public enum RangeType { /** Log-only read of one bucket over {@code [logStart, logStop)}. */ LOG, /** Full read of one bucket of a primary-key table: kv snapshot plus the log after it. */ PK_FULL, - /** Primary-key read merging the lake's splits for one bucket with the log tail after them. */ - UNION_PK + /** + * The log tail of one bucket of a primary-key table, replayed by key into the state it ended in. + * The rows before it are the lake's; this range contributes only what the lake does not hold yet. + */ + PK_TAIL } public static final String PROP_RANGE_TYPE = "fluss.range_type"; @@ -70,19 +80,10 @@ public enum RangeType { public static final String PROP_LOG_START_OFFSET = "fluss.log_start_offset"; public static final String PROP_LOG_STOP_OFFSET = "fluss.log_stop_offset"; public static final String PROP_KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; - public static final String PROP_LAKE_SNAPSHOT_ID = "fluss.lake_snapshot_id"; - public static final String PROP_LAKE_SPLITS = "fluss.lake_splits"; /** {@code kv_snapshot_id} for a bucket that has never been snapshotted. */ public static final long NO_KV_SNAPSHOT = -1L; - /** - * Separator for {@link #PROP_LAKE_SPLITS}. Each element is base64, whose alphabet - * ({@code A-Za-z0-9+/=}) has no comma, so joining is unambiguous. The factory rejects any element - * that contains one rather than trusting the caller. - */ - private static final String LAKE_SPLIT_SEPARATOR = ","; - private final RangeType rangeType; private final Partition partition; private final int bucketId; @@ -123,30 +124,24 @@ public static FlussScanRange pkFull(Partition partition, int bucketId, long kvSn } /** - * A union primary-key range: this bucket's splits of lake snapshot {@code lakeSnapshotId}, merged - * with the log from {@code logStartOffset} (where that lake snapshot ended, exclusive) up to - * {@code logStopOffset}. Each element of {@code lakeSplits} is a base64-encoded serialized lake - * split. + * The log tail of one bucket of a primary-key table: the change log over + * {@code [logStartOffset, logStopOffset)}, which the scanner replays by key and returns as the state + * that range ended in. {@code logStartOffset} is where the lake snapshot this tail follows left off, + * so the lake half holds everything before it. + * + *

An empty range is refused rather than planned: a bucket whose lake has caught up with its log + * contributes nothing, and the caller decides that by not planning a range at all. Reaching here + * with one would mean the two halves were bounded by different offsets. */ - public static FlussScanRange unionPk(Partition partition, int bucketId, long lakeSnapshotId, - List lakeSplits, long logStartOffset, long logStopOffset) { - Objects.requireNonNull(lakeSplits, "lakeSplits"); - if (lakeSplits.isEmpty()) { - throw new IllegalArgumentException( - "a UNION_PK range needs at least one lake split; a bucket with none is a plain log read"); - } - for (String split : lakeSplits) { - if (split == null || split.contains(LAKE_SPLIT_SEPARATOR)) { - // Would silently split one entry into two on the scanner side. - throw new IllegalArgumentException( - "lake split is not base64 (null or contains '" + LAKE_SPLIT_SEPARATOR + "'): " + split); - } + public static FlussScanRange pkTail(Partition partition, int bucketId, + long logStartOffset, long logStopOffset) { + if (logStartOffset >= logStopOffset) { + throw new IllegalArgumentException("a PK_TAIL range must read something, but bucket " + bucketId + + " was given [" + logStartOffset + ", " + logStopOffset + ")"); } - Map props = baseProps(RangeType.UNION_PK, partition, bucketId, + Map props = baseProps(RangeType.PK_TAIL, partition, bucketId, logStartOffset, logStopOffset); - props.put(PROP_LAKE_SNAPSHOT_ID, String.valueOf(lakeSnapshotId)); - props.put(PROP_LAKE_SPLITS, String.join(LAKE_SPLIT_SEPARATOR, lakeSplits)); - return new FlussScanRange(RangeType.UNION_PK, partition, bucketId, props); + return new FlussScanRange(RangeType.PK_TAIL, partition, bucketId, props); } private static Map baseProps(RangeType rangeType, Partition partition, diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java new file mode 100644 index 00000000000000..e9ddf4e8bbc5e7 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * A lake split of a primary-key table, with the log tail that supersedes part of it. + * + *

A tiered primary-key table is read as its lake plus the change log written since the lake snapshot. + * The two halves overlap by key, not by row: a row the lake holds may since have been updated or deleted + * in the log, and returning both copies is wrong in a way no row count reveals. This range is the lake + * half of that arrangement — the sibling connector's own split, unchanged in every respect that decides + * how it is read and scheduled — carrying the one extra fact BE needs: which slice of which bucket's log + * may contradict it. BE reads that slice's keys once per bucket and drops the lake rows they name; the + * surviving state of the tail is contributed separately, by a {@link FlussScanRange.RangeType#PK_TAIL} + * range, so each row is produced exactly once. + * + *

Delegation is the whole design. Everything except the format name and the extra payload is + * the sibling's answer, because everything else is what makes the lake half worth having: its file path + * and byte extent (native columnar readers), its split weight and target size (scheduling), its deletion + * vectors, its schema id, its partition values. Overriding any of them here would silently reshape the + * sibling's own read. That is also why this class must not grow a "smarter" version of any delegated + * method — the one thing it knows better than the sibling is the tail. + */ +public class FlussSuppressedLakeRange implements ConnectorScanRange { + + private static final long serialVersionUID = 1L; + + /** + * The {@code table_format_type} BE dispatches on. It is neither {@code fluss} nor {@code paimon}: the + * range is read by the sibling's reader stack WRAPPED in the suppression, and BE picks that composite + * reader by this name. Both plain names would land on a reader that ignores half the payload. + */ + public static final String TABLE_FORMAT_TYPE = "fluss_union"; + + /** {@code fluss.range_type} of a wrapped lake split; read by BE's C++ side, never by the scanner. */ + public static final String RANGE_TYPE_LAKE_SUPPRESS = "LAKE_SUPPRESS"; + + /** The log slice whose keys suppress rows of this split, as {@code partitionId:bucket:start:stop}. */ + public static final String PROP_TAIL = "fluss.union.tail"; + + private final ConnectorScanRange inner; + private final Tail tail; + + public FlussSuppressedLakeRange(ConnectorScanRange inner, Tail tail) { + this.inner = Objects.requireNonNull(inner, "inner"); + this.tail = Objects.requireNonNull(tail, "tail"); + } + + /** The wrapped sibling range, for a caller that has to look at what the lake half actually is. */ + public ConnectorScanRange getInner() { + return inner; + } + + public Tail getTail() { + return tail; + } + + /** + * One bucket's log tail: the half-open offset range that the lake snapshot does not cover yet. + * + *

Its encoding is also its identity on the BE side — the same string keys the cache that holds the + * suppression keys, so every split of the same bucket shares one read of the tail. Partition id is + * empty for an unpartitioned table rather than a sentinel number, so that the key of an unpartitioned + * table's bucket cannot collide with a partitioned one's. + */ + public static final class Tail implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + + private final FlussScanRange.Partition partition; + private final int bucketId; + private final long startOffset; + private final long stopOffset; + + public Tail(FlussScanRange.Partition partition, int bucketId, long startOffset, long stopOffset) { + this.partition = Objects.requireNonNull(partition, "partition"); + if (startOffset >= stopOffset) { + throw new IllegalArgumentException("a suppressing tail must contain something, but bucket " + + bucketId + " was given [" + startOffset + ", " + stopOffset + ")"); + } + this.bucketId = bucketId; + this.startOffset = startOffset; + this.stopOffset = stopOffset; + } + + public int getBucketId() { + return bucketId; + } + + public long getStartOffset() { + return startOffset; + } + + public long getStopOffset() { + return stopOffset; + } + + String encode() { + return (partition.isPartitioned() ? String.valueOf(partition.getId()) : "") + + ":" + bucketId + ":" + startOffset + ":" + stopOffset; + } + + @Override + public String toString() { + return "Tail{" + encode() + "}"; + } + } + + // ------------------------------------------------------------------ the two things that differ + + @Override + public String getTableFormatType() { + return TABLE_FORMAT_TYPE; + } + + /** + * The sibling fills its own payload first — serialized split or schema id and deletion files, file + * format, columns from path — and the tail is appended beside it. The two ride in different fields of + * the same descriptor ({@code paimon_params} and {@code fluss_params}), so neither has to know about + * the other. + */ + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + inner.populateRangeParams(formatDesc, rangeDesc); + Map params = new LinkedHashMap<>(); + params.put(FlussScanRange.PROP_RANGE_TYPE, RANGE_TYPE_LAKE_SUPPRESS); + params.put(PROP_TAIL, tail.encode()); + formatDesc.setFlussParams(params); + } + + // ------------------------------------------------------------------ everything else is the sibling's + + @Override + public Optional getPath() { + return inner.getPath(); + } + + @Override + public long getStart() { + return inner.getStart(); + } + + @Override + public long getLength() { + return inner.getLength(); + } + + @Override + public long getFileSize() { + return inner.getFileSize(); + } + + @Override + public String getFileFormat() { + return inner.getFileFormat(); + } + + @Override + public long getModificationTime() { + return inner.getModificationTime(); + } + + @Override + public long getSelfSplitWeight() { + return inner.getSelfSplitWeight(); + } + + @Override + public long getTargetSplitSize() { + return inner.getTargetSplitSize(); + } + + @Override + public List getHosts() { + return inner.getHosts(); + } + + @Override + public Map getProperties() { + return inner.getProperties(); + } + + @Override + public Map getPartitionValues() { + return inner.getPartitionValues(); + } + + @Override + public boolean isPartitionBearing() { + return inner.isPartitionBearing(); + } + + @Override + public long getPushDownRowCount() { + return inner.getPushDownRowCount(); + } + + @Override + public boolean isNativeReadRange() { + return inner.isNativeReadRange(); + } + + @Override + public String toString() { + return "FlussSuppressedLakeRange{" + tail + ", " + inner + "}"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java index 007319ec4293ae..c4513c97e8a1c8 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java @@ -22,6 +22,8 @@ import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; import java.util.ArrayList; import java.util.Collections; @@ -46,8 +48,12 @@ * a union read gets its catalog configuration. Nothing else in Doris supplies it — the Doris catalog is * configured with fluss bootstrap servers only. * - *

The column schema is deliberately NOT here. It is the one part that a statement re-reads through - * {@link FlussStatementScope}, so the handle stays a small, serializable identity object. + *

The column schema is deliberately NOT here — with one exception. It is the one part that a + * statement re-reads through {@link FlussStatementScope}, so the handle stays a small, serializable + * identity object; {@link #getKeyColumnTypes()} carries the types of the primary-key and partition-key + * columns alone, because planning has to reason about those by type (see its javadoc) and re-reading + * the whole schema for them would both cost a round trip and risk answering from a different schema + * version than the rest of this handle describes. */ public class FlussTableHandle implements ConnectorTableHandle { @@ -66,11 +72,12 @@ public class FlussTableHandle implements ConnectorTableHandle { /** The lake format's fluss name ({@code "paimon"}), or {@code null} when the table declares none. */ private final String dataLakeFormat; private final Map properties; + private final Map keyColumnTypes; public FlussTableHandle(String databaseName, String tableName, long tableId, int schemaId, boolean hasPrimaryKey, List primaryKeys, List bucketKeys, int bucketCount, List partitionKeys, boolean dataLakeEnabled, String dataLakeFormat, - Map properties) { + Map properties, Map keyColumnTypes) { this.databaseName = Objects.requireNonNull(databaseName, "databaseName"); this.tableName = Objects.requireNonNull(tableName, "tableName"); this.tableId = tableId; @@ -85,6 +92,9 @@ public FlussTableHandle(String databaseName, String tableName, long tableId, int this.properties = properties == null ? Collections.emptyMap() : Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + this.keyColumnTypes = keyColumnTypes == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(keyColumnTypes)); } /** Snapshots {@code tableInfo} into a handle. */ @@ -103,7 +113,26 @@ public static FlussTableHandle of(TableInfo tableInfo) { tableInfo.getPartitionKeys(), tableInfo.getTableConfig().isDataLakeEnabled(), lakeFormat == null ? null : lakeFormat.toString(), - tableInfo.getProperties().toMap()); + tableInfo.getProperties().toMap(), + keyColumnTypes(tableInfo)); + } + + /** + * The types of the columns that are part of the primary key or of the partition key, taken from the + * same {@link TableInfo} as every other field here. + */ + private static Map keyColumnTypes(TableInfo tableInfo) { + RowType rowType = tableInfo.getRowType(); + Map types = new LinkedHashMap<>(); + List keyColumns = new ArrayList<>(tableInfo.getPrimaryKeys()); + keyColumns.addAll(tableInfo.getPartitionKeys()); + for (String column : keyColumns) { + int index = rowType.getFieldIndex(column); + if (index >= 0) { + types.put(column, rowType.getTypeAt(index)); + } + } + return types; } public TablePath toTablePath() { @@ -162,6 +191,30 @@ public Map getProperties() { return properties; } + /** + * The fluss types of the primary-key and partition-key columns, by column name. + * + *

Split planning needs these two, and only these two, by type. A primary-key table read as the + * union of its lake and its log tail identifies rows across the two halves BY KEY, so a key column + * whose values do not compare exactly the same way on both sides (a float, a timestamp Doris rounds) + * cannot be read that way at all. A partition column is matched the same way one level up: a lake + * split is bound to a fluss partition by comparing the rendered partition values, which is only + * sound for a type both sides render identically. + * + *

Everything else about the schema stays out of the handle — the point is not "some of the + * schema", it is the columns whose type decides how the table can be PLANNED. + */ + public Map getKeyColumnTypes() { + return keyColumnTypes; + } + + /** The primary-key columns that are not partition columns — what a bucket's rows are keyed by. */ + public List getPhysicalPrimaryKeys() { + List physical = new ArrayList<>(primaryKeys); + physical.removeAll(partitionKeys); + return Collections.unmodifiableList(physical); + } + /** * Identity is the table plus the schema version it was read at: two handles for the same table at * different schema versions describe different column sets and must not compare equal. The diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java new file mode 100644 index 00000000000000..40b6051694e9f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.TimestampType; + +/** + * Which column types a primary-key table can be read by, when its lake and its log tail are read as one. + * + *

Such a read identifies rows across two halves that were written by different systems and are read + * by different code: the lake half arrives through paimon's readers, the log tail through fluss's. A + * row of the lake is dropped when its KEY appears in the tail, so the two sides must agree on what + * "the same key" means, exactly, for every key column. Where they might not, the table is not read that + * way at all — the fluss-only read still returns every row of a primary-key table, so refusing to + * combine the halves costs speed, never correctness. + * + *

The same question one level up decides which partition a lake split belongs to: the split's + * partition VALUES are compared, as text, with the ones fluss reports. That comparison is only sound + * for a type the two sides render identically, which is why partition columns are held to a stricter + * rule than key columns. + * + *

Neither rule is a fluss restriction — fluss accepts a DOUBLE primary key and an INT partition key + * quite happily ({@code TableDescriptorValidation}, whose only key-type ban is ARRAY/MAP/ROW). They are + * this connector's, and they exist because a wrong answer here is silent: an over-matched key drops a + * row that should have been returned, an under-matched one returns a superseded row twice. + */ +final class FlussUnionKeyTypes { + + /** Doris DATETIMEV2 stops at microseconds, so a finer fluss timestamp is rounded on the way in. */ + private static final int MAX_TIMESTAMP_PRECISION = 6; + + private FlussUnionKeyTypes() { + } + + /** + * Why {@code type} cannot be a key column of a union read, or null when it can. + * + *

The switch is exhaustive over fluss's type roots on purpose: a type a future fluss release adds + * lands in the default branch and is refused, rather than being waved through by a rule written + * before it existed. {@link FlussUnionKeyTypesTest} fails the build when that happens, so the + * refusal is a stopgap, not the answer. + */ + static String keyColumnRejection(DataType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case DECIMAL: + case CHAR: + case STRING: + case BINARY: + case BYTES: + case DATE: + return null; + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampRejection(((TimestampType) type).getPrecision()); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampRejection(((LocalZonedTimestampType) type).getPrecision()); + case FLOAT: + case DOUBLE: + // Two encodings of the same number (0.0 and -0.0) compare equal while NaN compares equal + // to nothing, so "the key is in the tail" has no stable answer. Doris's own key-based + // table types refuse floating-point keys for the same reason. + return "its equality is not exact for floating-point values"; + case TIME_WITHOUT_TIME_ZONE: + // Doris has no type for it at all: the column already reads as UNSUPPORTED. + return "Doris cannot represent a fluss TIME column"; + default: + return "the fluss connector does not know how to compare values of this type"; + } + } + + private static String timestampRejection(int precision) { + return precision <= MAX_TIMESTAMP_PRECISION ? null + : "Doris rounds it to microseconds, which can make two different keys look like one"; + } + + /** + * Why {@code type} cannot be a partition column of a union read, or null when it can. + * + *

Only STRING. A partition value reaches this connector already rendered — fluss hands back the + * text it stored, paimon renders its own typed partition value — and nothing guarantees the two + * spellings of a number, a date or a padded CHAR agree. They cannot be normalized here either, + * without this connector taking on how paimon renders every type it has. + */ + static String partitionColumnRejection(DataType type) { + return type.getTypeRoot() == DataTypeRoot.STRING ? null + : "a lake split is matched to a fluss partition by its rendered partition value, which" + + " only STRING is guaranteed to render the same way on both sides"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java index a9aa8749c55f95..c1a74f5b38bd2b 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java @@ -95,12 +95,38 @@ public void unionReadModeRejectsUnknownValueAtCreateCatalog() { "message should list the accepted values, was: " + e.getMessage()); } + /** + * The ceiling on how much log tail BE may hold while reading a primary-key table together with its + * lake. Zero and negative are rejected rather than read as "no limit": that limit exists because BE + * is a long-lived process shared by every other query, and there is deliberately no way to say + * "unbounded". + */ + @Test + public void theTailCeilingDefaultsToTwoMillionRowsAndMustBePositive() { + Assertions.assertEquals(2_000_000L, FlussConnectorProperties.maxTailRows(props())); + Assertions.assertEquals(500L, FlussConnectorProperties.maxTailRows( + props(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, " 500 "))); + + for (String bad : new String[] {"0", "-1", "lots", ""}) { + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, bad); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(properties), + "accepted '" + bad + "' as a row ceiling"); + Assertions.assertTrue( + e.getMessage().contains(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS), + e.getMessage()); + } + } + @Test public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() { Map properties = props( FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", "fluss.client.security.protocol", "sasl", FlussConnectorProperties.UNION_READ_MODE, "required", + FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, "10", FlussConnectorProperties.ENABLE_MAPPING_VARBINARY, "true", FlussConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true", "type", "fluss", diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java index 21ee674fec7d04..dc53842dc59ad7 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java @@ -29,7 +29,6 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -104,56 +103,37 @@ public void earliestOffsetGoesOutVerbatim() { Assertions.assertEquals("-2", range.getProperties().get("fluss.log_start_offset")); } + /** + * A tail range is a log range in everything but name — same offsets, same keys — and that is the + * point: what differs is how the scanner treats what it reads (replay by key rather than row by row), + * which is exactly what the range type says. Carrying a kv snapshot id would be the bug: the rows + * before the tail come from the lake, not from a snapshot. + */ @Test - public void unionPkRangeCarriesLakeSplitsButNoKvSnapshot() { - FlussScanRange range = FlussScanRange.unionPk(DT_20260101, 2, 41L, - Arrays.asList("c3BsaXQtMQ==", "c3BsaXQtMg=="), 300L, 305L); + public void tailRangeCarriesTheLogWindowAndNoKvSnapshot() { + FlussScanRange range = FlussScanRange.pkTail(DT_20260101, 2, 300L, 305L); Map expected = new LinkedHashMap<>(); - expected.put("fluss.range_type", "UNION_PK"); + expected.put("fluss.range_type", "PK_TAIL"); expected.put("fluss.partition_id", "77"); expected.put("fluss.partition_name", "dt=20260101"); expected.put("fluss.bucket_id", "2"); expected.put("fluss.log_start_offset", "300"); expected.put("fluss.log_stop_offset", "305"); - expected.put("fluss.lake_snapshot_id", "41"); - expected.put("fluss.lake_splits", "c3BsaXQtMQ==,c3BsaXQtMg=="); Assertions.assertEquals(expected, range.getProperties()); } /** - * The lake splits ride in one string, comma separated, which is only unambiguous because base64 - * has no comma. This pins the premise rather than assuming it: every byte value, encoded, must - * stay comma-free. + * An empty tail must not become a range. The bucket it would describe is one the lake already holds + * entirely, and planning says so by not producing a range at all; one that got here would mean the + * lake half and the fluss half were bounded by different offsets, which is how rows get lost. */ @Test - public void base64NeverContainsTheLakeSplitSeparator() { - byte[] allBytes = new byte[256]; - for (int i = 0; i < 256; i++) { - allBytes[i] = (byte) i; - } - for (int start = 0; start < 256; start++) { - String encoded = Base64.getEncoder() - .encodeToString(Arrays.copyOfRange(allBytes, start, 256)); - Assertions.assertFalse(encoded.contains(","), - "base64 of bytes " + start + ".. contains a comma: " + encoded); - } - } - - /** A split that could be mis-split on the scanner side must fail here, not read half a split there. */ - @Test - public void lakeSplitContainingTheSeparatorIsRejected() { - IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, - () -> FlussScanRange.unionPk(DT_20260101, 0, 1L, - Arrays.asList("ok", "not,base64"), 0L, 1L)); - Assertions.assertTrue(e.getMessage().contains("not,base64"), e.getMessage()); - } - - /** A union range with nothing from the lake is a plain log read; producing one would double-read. */ - @Test - public void unionPkWithoutLakeSplitsIsRejected() { + public void emptyTailRangeIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.pkTail(DT_20260101, 0, 5L, 5L)); Assertions.assertThrows(IllegalArgumentException.class, - () -> FlussScanRange.unionPk(DT_20260101, 0, 1L, Collections.emptyList(), 0L, 1L)); + () -> FlussScanRange.pkTail(DT_20260101, 0, 6L, 5L)); } @Test @@ -246,8 +226,7 @@ public void partitionWithoutColumnValuesIsRejected() { public void rangeSurvivesJavaSerialization() throws Exception { for (FlussScanRange original : Arrays.asList( FlussScanRange.log(FlussScanRange.Partition.NONE, 3, 10L, 42L), - FlussScanRange.unionPk(DT_20260101, 2, 41L, - Collections.singletonList("c3BsaXQ="), 300L, 305L))) { + FlussScanRange.pkTail(DT_20260101, 2, 300L, 305L))) { FlussScanRange restored = roundTrip(original); Assertions.assertEquals(original.getProperties(), restored.getProperties()); @@ -316,8 +295,7 @@ public void propertiesAreImmutable() { List ranges = Arrays.asList( FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L), FlussScanRange.pkFull(DT_20260101, 0, 1L, 0L, 1L), - FlussScanRange.unionPk(DT_20260101, 0, 1L, - Collections.singletonList("c3BsaXQ="), 0L, 1L)); + FlussScanRange.pkTail(DT_20260101, 0, 0L, 1L)); for (FlussScanRange range : ranges) { Assertions.assertThrows(UnsupportedOperationException.class, () -> range.getProperties().put("fluss.bucket_id", "999")); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java index b766dc0d40c473..d465dad42d14aa 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -34,6 +34,7 @@ import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * Split planning driven entirely off recorded admin answers, which is the point: the states worth @@ -352,16 +354,13 @@ public void prunedPartitionsOfPrimaryKeyTablesAreNotEvenAskedAbout() { // ------------------------------------------- tiered primary-key tables: read from fluss alone /** - * Merging a primary-key table's lake with its change log is not implemented, and what replaces it is - * NOT the "whatever has not been tiered away" fallback a log table would get: fluss keeps such a table's - * state in full, so its kv snapshot plus the log after it is every row. The lake is not consulted at all - * — a plan that quietly asked it would produce these same ranges, so the calls are asserted too, and the - * sibling factory fails loud on its own if planning ever tries to build one. + * A primary-key table whose tiering has never committed is read from fluss alone, and what replaces the + * union is NOT the "whatever has not been tiered away" fallback a log table would get: fluss keeps such + * a table's state in full, so its kv snapshot plus the log after it is every row. */ @Test - public void tieredPrimaryKeyTableIsReadFromFlussAlone() { + public void tieredPrimaryKeyTableWithoutALakeSnapshotIsReadFromFlussAlone() { registerTieredPkTable(2); - adminOps.readableLakeSnapshot = new LakeSnapshot(7L, Collections.emptyMap()); kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); latestOffsets(null, 12L, 25L); @@ -370,8 +369,6 @@ public void tieredPrimaryKeyTableIsReadFromFlussAlone() { Assertions.assertEquals(2, ranges.size()); assertPkRange(ranges.get(0), 0, 4L, 10L, 12L); assertPkRange(ranges.get(1), 1, 5L, 20L, 25L); - Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), - adminOps.calls.toString()); } /** @@ -393,23 +390,21 @@ public void theFallbackIsTheReadDisabledModeAsksForOutright() { /** * {@code required} is what a regression test sets so that it cannot pass without the lake having been - * read. No primary-key table can satisfy it today, so it fails loud instead of falling back — and it does - * so whether or not tiering has committed anything, because waiting for that commit would not help. + * read, so a primary-key table whose tiering has not committed fails loud rather than falling back — + * the same answer a log table gets, for the same reason. */ @Test - public void requiredModeRefusesATieredPrimaryKeyTable() { + public void requiredModeRefusesAPrimaryKeyTableWithNothingInItsLake() { registerTieredPkTable(2); DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); - Assertions.assertTrue(e.getMessage().contains("not implemented yet"), e.getMessage()); - Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), - adminOps.calls.toString()); + Assertions.assertTrue(e.getMessage().contains("no readable lake snapshot yet"), e.getMessage()); } /** * The plan's own account of not having read the lake. Nothing else in the plan of a tiered primary-key - * table distinguishes it from one that was read as a union, which is what the follow-up work will do. + * table distinguishes it from one that was read as a union. */ @Test public void explainShowsATieredPrimaryKeyTableWasReadWithoutItsLake() { @@ -422,9 +417,8 @@ public void explainShowsATieredPrimaryKeyTableWasReadWithoutItsLake() { StringBuilder output = new StringBuilder(); provider.appendExplainInfo(output, "", Collections.emptyMap()); - Assertions.assertEquals( - "flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=1, mode=auto\n", - output.toString()); + Assertions.assertEquals("flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0," + + " logRanges=0, pkRanges=1, pkTailRanges=0, mode=auto\n", output.toString()); } @@ -640,8 +634,8 @@ public void explainReportsTheUnionAndTheSizeOfEachHalf() { provider.appendExplainInfo(output, "", Collections.emptyMap()); Assertions.assertEquals( - "flussScan: unionRead=yes, lakeSplits=3, logRanges=1, pkRanges=0, mode=auto\n", - output.toString()); + "flussScan: unionRead=yes, lakeSplits=3, suppressedLakeSplits=0, logRanges=1," + + " pkRanges=0, pkTailRanges=0, mode=auto\n", output.toString()); } /** Nothing in the lake means the log holds the whole table, so the fluss-only read IS complete. */ @@ -678,8 +672,495 @@ public void disabledModeReadsTheLogWithoutAskingAboutTheLake() { adminOps.calls.toString()); } + // ------------------------------------------- union read of a primary-key table: lake + log tail + + /** + * The shape of the whole arrangement, in one plan. Bucket 0 has been tiered and has written more + * since, so its lake split is bound to the tail that supersedes part of it AND the tail is planned as + * its own range — the lake split only hides rows, it never produces the new ones. Bucket 1 has been + * tiered and written nothing since, so its lake split is passed through untouched and fluss + * contributes nothing at all. + */ + @Test + public void bucketWithATailBindsItToItsLakeSplitsAndPlansTheTailOnce() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 200L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(1)); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertSuppressed(ranges.get(0), 0, 100L, 105L); + Assertions.assertTrue(ranges.get(1) instanceof RecordingLakeSibling.LakeRange, + "bucket 1 has no tail and must be passed through: " + ranges.get(1)); + assertTailRange(ranges.get(2), 0, 100L, 105L); + } + + /** + * The suppressing tail and the range that produces the tail's rows must be the SAME window. Any gap + * between them is a wrong answer in one direction or the other: a suppression window wider than the + * produced one hides rows nothing brings back, a narrower one lets a superseded row through beside + * its replacement. + */ + @Test + public void theSuppressedWindowIsExactlyTheWindowTheTailRangeReads() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 130L); + earliestOffsets(null, 7L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + FlussSuppressedLakeRange.Tail tail = ((FlussSuppressedLakeRange) ranges.get(0)).getTail(); + Map tailRange = ranges.get(1).getProperties(); + Assertions.assertEquals(String.valueOf(tail.getStartOffset()), + tailRange.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(tail.getStopOffset()), + tailRange.get("fluss.log_stop_offset")); + } + + /** + * A bucket the lake has never seen is read whole from fluss, exactly as it would be with no lake at + * all — and nothing of it can be suppressed, because the lake holds none of it. + */ + @Test + public void bucketTheLakeHasNeverSeenIsReadWholeFromFluss() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, null)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 100L, 25L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertTrue(ranges.get(0) instanceof RecordingLakeSibling.LakeRange, "bucket 0 is tiered" + + " up to where its log ends and must be passed through: " + ranges.get(0)); + assertPkRange(ranges.get(1), 1, 5L, 20L, 25L); + } + + /** + * The tail of one bucket must not be bound to another bucket's split. Bucket-blind binding still + * produces a plausible plan — every split suppressed by SOME tail — and returns wrong rows: the keys + * of bucket 0's tail say nothing about the rows of bucket 1. + */ + @Test + public void eachBucketsSplitsAreBoundToThatBucketsTail() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 250L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(1), + RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + assertSuppressed(ranges.get(0), 1, 200L, 250L); + assertSuppressed(ranges.get(1), 0, 100L, 105L); + } + + /** The same, one level up: a partition's splits are bound to the tails of THAT partition's buckets. */ + @Test + public void eachPartitionsSplitsAreBoundToThatPartitionsTails() { + registerPartitionedPkLakeTable(1, "20260101", "20260102"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets( + new long[] {100L}, new long[] {700L})); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + kvSnapshots("20260102", new long[] {2L}, new long[] {20L}); + latestOffsets("20260101", 105L); + latestOffsets("20260102", 750L); + earliestOffsets("20260101", 0L); + earliestOffsets("20260102", 0L); + lakeSplits( + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20260102")), + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20260101"))); + + List ranges = plan(PK_TABLE, catalog()); + + assertSuppressed(ranges.get(0), 0, 700L, 750L); + assertSuppressed(ranges.get(1), 0, 100L, 105L); + } + + /** + * A lake split of a partition this scan does not read from fluss — one fluss has dropped, or one the + * engine pruned away — has no tail to be bound to and is passed through. It cannot be dropped either: + * partition pruning removes the fluss half of a partition, not the predicate that pruned it. + */ + @Test + public void lakeSplitOfAPartitionThisScanDoesNotReadIsPassedThrough() { + registerPartitionedPkLakeTable(1, "20260101"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets(new long[] {100L})); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + latestOffsets("20260101", 105L); + earliestOffsets("20260101", 0L); + lakeSplits( + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20251231"))); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertTrue(ranges.get(0) instanceof RecordingLakeSibling.LakeRange, ranges.toString()); + assertTailRange(ranges.get(1), 0, 100L, 105L); + } + + /** + * The bucket a split holds is the one fact this connector cannot work out for itself, and an older + * paimon plugin does not report it. Reading that absence as "this bucket has no tail" would return + * every superseded row a second time, so it is an error instead. + */ + @Test + public void lakeSplitThatDoesNotSayWhichBucketItHoldsIsRefused() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.withoutABucket()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("paimon.bucket"), e.getMessage()); + } + + /** A lake table bucketed differently from the fluss table cannot be matched to it bucket by bucket. */ + @Test + public void lakeSplitInABucketThisTableDoesNotHaveIsRefused() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(7)); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("bucketed alike"), e.getMessage()); + } + + /** + * The lake holding data for a bucket fluss has no tiering offset for means the two disagree about what + * has been tiered. Passing the split through would return that bucket's rows twice — once from the + * lake, once from the {@code PK_FULL} range the missing offset produces. + */ + @Test + public void lakeSplitOfABucketFlussRecordsNoTieringOffsetForIsRefused() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, null)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 25L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(1)); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("metadata disagrees"), e.getMessage()); + } + + /** + * Fluss deletes old log segments on a timer that does not wait for tiering, so the beginning of a tail + * can be gone. There is nowhere else to read it from — the lake's copy stops exactly where the missing + * tail begins — so {@code auto} gives up the lake half rather than return fewer rows than the table + * holds, and says so in EXPLAIN. + */ + @Test + public void tailTheLogNoLongerHoldsGivesUpTheLakeHalf() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 0L, 201L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + assertPkRange(ranges.get(1), 1, 5L, 20L, 205L); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("unionRead=no"), output.toString()); + Assertions.assertTrue(output.toString().contains("degraded=tail-truncated"), output.toString()); + } + + /** A bucket fluss will not report an earliest offset for is not "fine", it is unverifiable. */ + @Test + public void tailThatCannotBeVerifiedGivesUpTheLakeHalf() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + adminOps.earliestOffsetsByPartition.put(null, Collections.emptyMap()); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(1, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + } + + /** + * What replaces the lake half is the very read {@code disabled} asks for outright, down to the ranges. + * That is what makes it safe to give up so late: a primary-key table read from fluss alone is the whole + * table. + */ + @Test + public void whatReplacesTheLakeHalfIsTheReadDisabledModeAsksForOutright() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 104L, 0L); + + List> degraded = rangeProperties(plan(PK_TABLE, catalog())); + List> disabled = rangeProperties( + plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + Assertions.assertEquals(disabled, degraded); + } + + /** Under {@code required} the same truncated tail is an error: there is nothing to fall back to. */ + @Test + public void requiredModeRefusesATailTheLogNoLongerHolds() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 101L); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("the log now starts at 101"), e.getMessage()); + } + + /** + * A key column Doris cannot compare exactly is a permanent property of the table, so it is settled + * before anything is asked of the lake — that is what lets the answer be the same at plan-translation + * time, when the key columns are kept in the scan's tuple, as it is here. + */ + @Test + public void keyColumnThatCannotBeComparedExactlyGivesUpTheLakeHalf() { + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("degraded=key-type"), output.toString()); + // Settled without asking the lake anything at all. + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + @Test + public void requiredModeRefusesAKeyColumnThatCannotBeComparedExactly() { + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("primary-key column 'id'"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("floating-point"), e.getMessage()); + } + + /** + * A lake split is matched to a fluss partition by comparing rendered partition values, so a partition + * column of a type the two sides may spell differently is refused the same way. Fluss itself allows + * such a column — INT, DATE, BOOLEAN are all legal partition keys — so this is not a case the schema + * makes impossible. + */ + @Test + public void partitionColumnThatMayNotRenderAlikeGivesUpTheLakeHalf() { + registerPartitionedPkLakeTable(1, DataTypes.INT(), "20260101"); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + latestOffsets("20260101", 105L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, ranges.size()); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("degraded=partition-type"), output.toString()); + } + + @Test + public void requiredModeRefusesAPartitionColumnThatMayNotRenderAlike() { + registerPartitionedPkLakeTable(1, DataTypes.INT(), "20260101"); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("partition column 'dt'"), e.getMessage()); + } + + /** + * The kv snapshots are read for every partition even when the lake half turns out to cover it, because + * the fallback needs them and reading them after the offsets would leave a bucket bounded by an offset + * older than the snapshot it is read from. The order is the assertion. + */ + @Test + public void snapshotsAreReadBeforeTheOffsetsEvenWhenTheLakeCoversEverything() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + plan(PK_TABLE, catalog()); + + int snapshotCall = indexOfCall("getLatestKvSnapshots"); + int latestCall = indexOfCall("listOffsets(db.pk_tbl, [0], LatestSpec)"); + int earliestCall = indexOfCall("listOffsets(db.pk_tbl, [0], EarliestSpec)"); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < latestCall, adminOps.calls.toString()); + Assertions.assertTrue(latestCall < earliestCall, adminOps.calls.toString()); + } + + /** EXPLAIN accounts for all three parts, so a regression test can tell which of them did the work. */ + @Test + public void explainCountsTheSuppressedSplitsAndTheTailsApart() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 200L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(1)); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals("flussScan: unionRead=yes, lakeSplits=3, suppressedLakeSplits=2," + + " logRanges=0, pkRanges=0, pkTailRanges=1, mode=auto\n", output.toString()); + } + + // ------------------------------------------- the key columns BE has to read either way + + /** + * BE identifies the rows a tail supersedes by their keys, so those columns have to be read whether or + * not the query asked for them. The engine keeps them in the scan's tuple on the strength of this + * answer; the projection above the scan drops them again before the user sees a row. + */ + @Test + public void unionReadKeepsTheKeyColumnsInTheScan() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Assertions.assertEquals(Collections.singleton("id"), mustReadColumns(PK_TABLE, catalog())); + } + + /** Only the physical key: a partition column is the same for every row of a bucket. */ + @Test + public void thePartitionColumnsAreNotPartOfTheKeyBeMustRead() { + registerPartitionedPkLakeTable(1, "20260101"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets(new long[] {100L})); + + Assertions.assertEquals(Collections.singleton("id"), mustReadColumns(PK_TABLE, catalog())); + } + + /** + * Nothing else keeps a column it was not asked for. Each of these reads is served by one scanner that + * needs no key at all, so keeping one would be a column read for nobody — and, for a log table, would + * pull the lake snapshot's round trip forward into plan translation for nothing. + */ + @Test + public void everyOtherReadKeepsNothing() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(1L, 1L)); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(LOG_TABLE, catalog())); + Assertions.assertTrue(adminOps.calls.isEmpty(), adminOps.calls.toString()); + + registerPkTable(PK_TABLE, 1); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(PK_TABLE, catalog())); + + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + Assertions.assertEquals(Collections.emptySet(), + mustReadColumns(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(PK_TABLE, catalog())); + } + + /** + * The engine asks for the key columns while translating the plan and plans the ranges later, and the + * two answers must come from ONE resolution. Were they resolved twice, a snapshot committed in between + * could make the first say "no lake" (so the key columns are pruned away) and the second say "lake", + * leaving BE to look for a key column that is not in its projection. + */ + @Test + public void bothQuestionsAreAnsweredByTheSameResolution() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + provider.getMustReadColumns(session, handle(PK_TABLE)); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("getReadableLakeSnapshot")).count(), + adminOps.calls.toString()); + } + // ---------------------------------------------------------------- what BE and EXPLAIN receive + /** + * What BE needs to suppress by key, and only on the read that suppresses: the key columns by name (the + * types travel as ordinary slot descriptors, because the columns are ordinary projected columns) and + * the ceiling on how much tail it may hold while doing it. + */ + @Test + public void primaryKeyUnionTellsBeWhichColumnsTheKeyIsMadeOf() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Map props = nodeProperties(PK_TABLE, catalog()); + + Assertions.assertEquals("id", props.get("fluss.union.pk_names")); + Assertions.assertEquals("2000000", props.get("fluss.union.max_tail_rows")); + } + + @Test + public void theTailCeilingIsTheOneTheCatalogDeclares() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Map props = nodeProperties(PK_TABLE, + catalog(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, "500")); + + Assertions.assertEquals("500", props.get("fluss.union.max_tail_rows")); + } + + /** A read that suppresses nothing must not describe a key: there is no reader on the far side. */ + @Test + public void readThatSuppressesNothingSendsNoKey() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(1L)); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertFalse(props.containsKey("fluss.union.pk_names"), props.toString()); + Assertions.assertFalse(props.containsKey("fluss.union.max_tail_rows"), props.toString()); + } + @Test public void scanLevelParamsCarryTheClientConfigAndTableIdentity() { registerLogTable(LOG_TABLE, 1); @@ -729,8 +1210,8 @@ public void explainReportsHowTheScanWasActuallyPlanned() { provider.appendExplainInfo(output, " ", Collections.emptyMap()); Assertions.assertEquals( - " flussScan: unionRead=no, lakeSplits=0, logRanges=2, pkRanges=0, mode=auto\n", - output.toString()); + " flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0, logRanges=2," + + " pkRanges=0, pkTailRanges=0, mode=auto\n", output.toString()); } /** @@ -750,8 +1231,8 @@ public void explainCountsPrimaryKeyRangesApartFromLogRanges() { provider.appendExplainInfo(output, "", Collections.emptyMap()); Assertions.assertEquals( - "flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=2, mode=auto\n", - output.toString()); + "flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0, logRanges=0," + + " pkRanges=2, pkTailRanges=0, mode=auto\n", output.toString()); } @Test @@ -809,6 +1290,11 @@ private Map nodeProperties(TablePath tablePath, Map mustReadColumns(TablePath tablePath, Map catalogProperties) { + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling) + .getMustReadColumns(session, handle(tablePath)); + } + private static ConnectorScanRequest request(ConnectorTableHandle handle, List requiredPartitions) { return ConnectorScanRequest.builder(handle, Collections.emptyList()) .requiredPartitions(requiredPartitions) @@ -913,11 +1399,24 @@ private void kvSnapshots(String partitionName, long[] snapshotIds, long[] logOff /** Latest offsets for buckets 0..n-1 of {@code partitionName} ({@code null} = unpartitioned). */ private void latestOffsets(String partitionName, long... offsets) { + adminOps.latestOffsetsByPartition.put(partitionName, byBucket(offsets)); + } + + /** + * Earliest offsets for buckets 0..n-1 — how far back fluss can still serve. Only a union read of a + * primary-key table asks for these, which is why the log-table fixtures do not set them: a test that + * needed them and did not say so gets an error from the recorder, not a default. + */ + private void earliestOffsets(String partitionName, long... offsets) { + adminOps.earliestOffsetsByPartition.put(partitionName, byBucket(offsets)); + } + + private static Map byBucket(long... offsets) { Map byBucket = new LinkedHashMap<>(); for (int bucket = 0; bucket < offsets.length; bucket++) { byBucket.put(bucket, offsets[bucket]); } - adminOps.latestOffsetsByPartition.put(partitionName, byBucket); + return byBucket; } private static void assertLogRange(ConnectorScanRange range, int bucket, long start, long stop) { @@ -1006,6 +1505,93 @@ private static Map offsets(Long... byBucket) { return offsets; } + /** The exact splits the sibling's scan planner returns, in order. */ + private void lakeSplits(RecordingLakeSibling.LakeRange... splits) { + sibling().lakeRanges = new ArrayList<>(Arrays.asList(splits)); + } + + /** Lake offsets for buckets 0..n-1 of each partition, partition ids 100, 101, ... in order. */ + private static Map partitionedOffsets(long[]... byPartition) { + Map offsets = new HashMap<>(); + for (int partition = 0; partition < byPartition.length; partition++) { + for (int bucket = 0; bucket < byPartition[partition].length; bucket++) { + offsets.put(new TableBucket(FlussTestTables.TABLE_ID, 100L + partition, bucket), + byPartition[partition][bucket]); + } + } + return offsets; + } + + /** + * A primary-key table tiered into a lake whose lake IS read. The same table as + * {@link #registerTieredPkTable}, but declaring that reaching the sibling is expected — the other + * fixture keeps that guard on, for the tests that assert the lake was never consulted. + */ + private void registerPkLakeTable(int buckets) { + registerTieredPkTable(buckets); + siblingExpected = true; + } + + /** The same, keyed by a column of {@code keyType} — for the types a union read cannot compare. */ + private void registerPkLakeTableKeyedBy(DataType keyType) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", keyType.copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(1, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + siblingExpected = true; + } + + /** A partitioned primary-key lake table, partitioned by a STRING {@code dt}. */ + private void registerPartitionedPkLakeTable(int buckets, String... partitionValues) { + registerPartitionedPkLakeTable(buckets, DataTypes.STRING(), partitionValues); + } + + private void registerPartitionedPkLakeTable(int buckets, DataType partitionType, + String... partitionValues) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("dt", partitionType.copy(false)) + .primaryKey("id", "dt") + .partitionedBy("dt") + .buckets(buckets, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(PK_TABLE, partitions); + siblingExpected = true; + } + + /** A lake split bound to the tail of {@code bucket} over {@code [start, stop)}. */ + private static void assertSuppressed(ConnectorScanRange range, int bucket, long start, long stop) { + Assertions.assertTrue(range instanceof FlussSuppressedLakeRange, + "expected a suppressed lake split but got " + range); + FlussSuppressedLakeRange.Tail tail = ((FlussSuppressedLakeRange) range).getTail(); + Assertions.assertEquals(bucket, tail.getBucketId()); + Assertions.assertEquals(start, tail.getStartOffset()); + Assertions.assertEquals(stop, tail.getStopOffset()); + } + + private static void assertTailRange(ConnectorScanRange range, int bucket, long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("PK_TAIL", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + /** {@code count} ranges for the sibling's scan planner to return as the lake half. */ private void lakeRanges(int count) { List ranges = new ArrayList<>(count); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java new file mode 100644 index 00000000000000..8fa40dcde1c67f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TPaimonFileDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The wrapper that carries a log tail alongside a lake split. + * + *

Everything it does NOT change is what makes the lake half worth having — its file extent, its split + * weight, its deletion vectors — so the delegation is asserted method by method against a stand-in that + * answers something distinctive for each. A silently swallowed answer (a zero weight, an empty path) + * would not fail any assertion about rows; it would just make the lake half read or schedule badly. + */ +public class FlussSuppressedLakeRangeTest { + + private static final FlussScanRange.Partition DT_20260101 = + FlussScanRange.Partition.of(77L, "dt=20260101", + Collections.singletonMap("dt", "20260101")); + + /** A sibling range whose every answer is distinctive, so a dropped delegation is visible. */ + private static final class LakeSplit implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + + @Override + public Optional getPath() { + return Optional.of("/warehouse/db/tbl/bucket-2/data-1.parquet"); + } + + @Override + public long getStart() { + return 4096L; + } + + @Override + public long getLength() { + return 8192L; + } + + @Override + public long getFileSize() { + return 16384L; + } + + @Override + public String getFileFormat() { + return "parquet"; + } + + @Override + public long getModificationTime() { + return 1735689600L; + } + + @Override + public long getSelfSplitWeight() { + return 7L; + } + + @Override + public long getTargetSplitSize() { + return 134217728L; + } + + @Override + public List getHosts() { + return Arrays.asList("be-1", "be-2"); + } + + @Override + public Map getProperties() { + Map props = new LinkedHashMap<>(); + props.put("paimon.bucket", "2"); + props.put("paimon.schema_id", "5"); + return props; + } + + @Override + public Map getPartitionValues() { + return Collections.singletonMap("dt", "20260101"); + } + + @Override + public boolean isPartitionBearing() { + return true; + } + + @Override + public long getPushDownRowCount() { + return 42L; + } + + @Override + public boolean isNativeReadRange() { + return true; + } + + @Override + public String getTableFormatType() { + return "paimon"; + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setPaimonParams(new TPaimonFileDesc()); + rangeDesc.setColumnsFromPathKeys(Collections.singletonList("dt")); + rangeDesc.setColumnsFromPath(Collections.singletonList("20260101")); + } + } + + private static FlussSuppressedLakeRange wrap() { + return new FlussSuppressedLakeRange(new LakeSplit(), + new FlussSuppressedLakeRange.Tail(DT_20260101, 2, 100L, 105L)); + } + + @Test + public void everythingThatDecidesHowTheSplitIsReadStaysTheSiblingsAnswer() { + FlussSuppressedLakeRange wrapped = wrap(); + ConnectorScanRange inner = new LakeSplit(); + + Assertions.assertEquals(inner.getPath(), wrapped.getPath()); + Assertions.assertEquals(inner.getStart(), wrapped.getStart()); + Assertions.assertEquals(inner.getLength(), wrapped.getLength()); + Assertions.assertEquals(inner.getFileSize(), wrapped.getFileSize()); + Assertions.assertEquals(inner.getFileFormat(), wrapped.getFileFormat()); + Assertions.assertEquals(inner.getModificationTime(), wrapped.getModificationTime()); + Assertions.assertEquals(inner.getSelfSplitWeight(), wrapped.getSelfSplitWeight()); + Assertions.assertEquals(inner.getTargetSplitSize(), wrapped.getTargetSplitSize()); + Assertions.assertEquals(inner.getHosts(), wrapped.getHosts()); + Assertions.assertEquals(inner.getProperties(), wrapped.getProperties()); + Assertions.assertEquals(inner.getPartitionValues(), wrapped.getPartitionValues()); + Assertions.assertEquals(inner.isPartitionBearing(), wrapped.isPartitionBearing()); + Assertions.assertEquals(inner.getPushDownRowCount(), wrapped.getPushDownRowCount()); + Assertions.assertEquals(inner.isNativeReadRange(), wrapped.isNativeReadRange()); + } + + /** + * The one answer that changes. BE picks the reader by this name, and both plain names would be wrong: + * {@code paimon} reads the split and ignores the tail (superseded rows come back), {@code fluss} reads + * the tail and cannot read the split at all. + */ + @Test + public void theFormatNameIsTheCompositeReadersOwn() { + Assertions.assertEquals("fluss_union", wrap().getTableFormatType()); + } + + /** + * The payload is both halves, in their own thrift fields: whatever the sibling wrote, untouched, plus + * the tail. Asserted whole rather than by spot check — this is an untyped string map that nothing + * between here and BE type-checks. + */ + @Test + public void theDescriptorCarriesTheSiblingsPayloadAndTheTail() { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + wrap().populateRangeParams(formatDesc, rangeDesc); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "LAKE_SUPPRESS"); + expected.put("fluss.union.tail", "77:2:100:105"); + Assertions.assertEquals(expected, formatDesc.getFlussParams()); + // The sibling's own payload has to survive beside it: the paimon reader fails outright without it. + Assertions.assertTrue(formatDesc.isSetPaimonParams()); + Assertions.assertEquals(Collections.singletonList("20260101"), rangeDesc.getColumnsFromPath()); + } + + /** + * An unpartitioned table's tail leaves the partition segment empty rather than writing a sentinel id. + * The string is also the cache key BE builds the suppression set under, so "no partition" and + * "partition -1" must not be the same key. + */ + @Test + public void anUnpartitionedTailLeavesThePartitionSegmentEmpty() { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + new FlussSuppressedLakeRange(new LakeSplit(), + new FlussSuppressedLakeRange.Tail(FlussScanRange.Partition.NONE, 3, 0L, 9L)) + .populateRangeParams(formatDesc, new TFileRangeDesc()); + + Assertions.assertEquals(":3:0:9", formatDesc.getFlussParams().get("fluss.union.tail")); + } + + /** An empty tail suppresses nothing; producing one would mean planning wrapped a split for no reason. */ + @Test + public void anEmptyTailIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussSuppressedLakeRange.Tail(DT_20260101, 0, 5L, 5L)); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java index 0b7be5b4d99677..e1b1ff3593dc3c 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.fluss; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -39,9 +41,12 @@ private static FlussTableHandle handle(String tableName, long tableId, int schem Map properties = new LinkedHashMap<>(); properties.put("table.datalake.enabled", "true"); properties.put("table.datalake.paimon.warehouse", "/tmp/lake"); + Map keyTypes = new LinkedHashMap<>(); + keyTypes.put("dt", DataTypes.STRING()); + keyTypes.put("id", DataTypes.INT()); return new FlussTableHandle("db", tableName, tableId, schemaId, true, Arrays.asList("dt", "id"), Collections.singletonList("id"), 4, - Collections.singletonList("dt"), true, "paimon", properties); + Collections.singletonList("dt"), true, "paimon", properties, keyTypes); } @Test @@ -64,9 +69,24 @@ public void survivesJavaSerializationWithEveryFieldIntact() { Assertions.assertTrue(restored.isDataLakeEnabled()); Assertions.assertEquals("paimon", restored.getDataLakeFormat()); Assertions.assertEquals("/tmp/lake", restored.getProperties().get("table.datalake.paimon.warehouse")); + // The key column types decide whether the table can be read as its lake plus its log at all, and + // they are the one part of the schema the handle carries, so they have to survive the trip too. + Assertions.assertEquals(DataTypes.INT(), restored.getKeyColumnTypes().get("id")); + Assertions.assertEquals(DataTypes.STRING(), restored.getKeyColumnTypes().get("dt")); Assertions.assertEquals(handle("pk_table", 7L, 3), restored); } + /** + * A bucket's rows are keyed by the primary key MINUS the partition columns: a bucket lives inside one + * partition, so the partition columns are the same for every row in it and carrying them would make + * the key wider than it is. Order follows the primary key's own. + */ + @Test + public void physicalPrimaryKeyDropsThePartitionColumns() { + Assertions.assertEquals(Collections.singletonList("id"), + handle("t", 1L, 1).getPhysicalPrimaryKeys()); + } + @Test public void identityIsTheTableAtItsSchemaVersion() { Assertions.assertEquals(handle("t", 1L, 1), handle("t", 1L, 1)); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java new file mode 100644 index 00000000000000..8056cf543ecf06 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Which columns a primary-key table can be read by when its lake and its log tail are read as one. + * + *

The property worth more than any single row is completeness: every one of fluss's type roots has a + * deliberate verdict here, so a type a future fluss release adds cannot slip in as "allowed by omission". + * The rules themselves are one-directional in their risk — allowing a type that does not compare + * identically on both sides returns wrong rows silently, refusing one that would have been fine only + * costs the lake's speed — so the verdicts below are written to be defensible in that direction. + */ +public class FlussUnionKeyTypesTest { + + /** One type per fluss type root, and the verdict each is meant to get as a KEY column. */ + private static Map keyVerdicts() { + Map expected = new LinkedHashMap<>(); + expected.put(DataTypes.BOOLEAN(), true); + expected.put(DataTypes.TINYINT(), true); + expected.put(DataTypes.SMALLINT(), true); + expected.put(DataTypes.INT(), true); + expected.put(DataTypes.BIGINT(), true); + expected.put(DataTypes.DECIMAL(20, 4), true); + expected.put(DataTypes.CHAR(10), true); + expected.put(DataTypes.STRING(), true); + expected.put(DataTypes.BINARY(16), true); + expected.put(DataTypes.BYTES(), true); + expected.put(DataTypes.DATE(), true); + expected.put(DataTypes.TIMESTAMP(6), true); + expected.put(DataTypes.TIMESTAMP_LTZ(6), true); + // Refused, each for its own reason: two encodings of one number and a NaN that equals nothing; + // a type Doris cannot represent at all; and the three fluss itself already refuses as a key, + // covered here so the switch stays exhaustive rather than relying on that. + expected.put(DataTypes.FLOAT(), false); + expected.put(DataTypes.DOUBLE(), false); + expected.put(DataTypes.TIME(), false); + expected.put(DataTypes.ARRAY(DataTypes.INT()), false); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), false); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), false); + return expected; + } + + @Test + public void everyFlussTypeRootHasADeliberateVerdict() { + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : keyVerdicts().entrySet()) { + DataType type = entry.getKey(); + covered.add(type.getTypeRoot()); + boolean allowed = FlussUnionKeyTypes.keyColumnRejection(type) == null; + Assertions.assertEquals(entry.getValue(), allowed, "key verdict for " + type); + // Every root is also asked the partition question, so neither switch can be the one that + // silently falls through on a type fluss adds. + Assertions.assertEquals(type.getTypeRoot() == DataTypeRoot.STRING, + FlussUnionKeyTypes.partitionColumnRejection(type) == null, + "partition verdict for " + type); + } + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "a fluss type root has no verdict here; decide whether it can be a union-read key"); + } + + /** + * Doris stops at microseconds, so a finer fluss timestamp arrives rounded — and two keys that differ + * only in the digits it rounded away become one. Over-matching drops a row that should have been + * returned, which no row count reveals. + */ + @Test + public void timestampFinerThanDorisCanHoldIsRefused() { + for (int precision = 0; precision <= 6; precision++) { + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP(precision)), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP_LTZ(precision)), + "TIMESTAMP_LTZ(" + precision + ")"); + } + for (int precision = 7; precision <= 9; precision++) { + Assertions.assertNotNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP(precision)), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNotNull( + FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP_LTZ(precision)), + "TIMESTAMP_LTZ(" + precision + ")"); + } + } + + /** + * Fluss accepts every one of these as a partition key ({@code PartitionUtils}), so refusing all but + * STRING is this connector's own rule, not a restriction inherited from below: a partition value + * arrives here already rendered, by two different systems, and only a string is guaranteed to come + * out the same on both sides. + */ + @Test + public void everyPartitionKeyTypeFlussAllowsButStringIsRefused() { + List flussAllows = Arrays.asList(DataTypes.CHAR(8), DataTypes.BOOLEAN(), + DataTypes.BINARY(4), DataTypes.BYTES(), DataTypes.TINYINT(), DataTypes.SMALLINT(), + DataTypes.INT(), DataTypes.BIGINT(), DataTypes.DATE(), DataTypes.TIME()); + for (DataType type : flussAllows) { + Assertions.assertNotNull(FlussUnionKeyTypes.partitionColumnRejection(type), type.toString()); + } + Assertions.assertNull(FlussUnionKeyTypes.partitionColumnRejection(DataTypes.STRING())); + } + + /** Nullability is not part of the question — a key column is NOT NULL in fluss either way. */ + @Test + public void nullabilityDoesNotChangeTheVerdict() { + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.INT().copy(false))); + Assertions.assertNull(FlussUnionKeyTypes.partitionColumnRejection(DataTypes.STRING().copy(false))); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java index 77cb83200270ab..35676e1f7b4ca6 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -62,6 +62,13 @@ class RecordingFlussAdminOps implements FlussAdminOps { * unpartitioned table) — what split planning stops each bucket at. */ final Map> latestOffsetsByPartition = new HashMap<>(); + /** + * Earliest log offset per bucket, keyed the same way — how far back fluss can still serve. Kept apart + * from the latest offsets rather than sharing one map: a union read of a primary-key table asks for + * both, and a fixture that answered the same numbers to both questions would make the guard that + * compares them ("is the tail still there?") pass no matter what it was given. + */ + final Map> earliestOffsetsByPartition = new HashMap<>(); /** * Latest kv snapshot per bucket, keyed by fluss's own partition name ({@code null} for an * unpartitioned table) — where primary-key planning starts each bucket's change log. @@ -189,9 +196,13 @@ private Map recordedOffsets(TablePath tablePath, String partition Collection buckets, OffsetSpec offsetSpec) { calls.add("listOffsets(" + tablePath + (partitionName == null ? "" : ", " + partitionName) + ", " + buckets + ", " + offsetSpec.getClass().getSimpleName() + ")"); - Map offsets = latestOffsetsByPartition.get(partitionName); + boolean earliest = offsetSpec instanceof OffsetSpec.EarliestSpec; + Map offsets = earliest + ? earliestOffsetsByPartition.get(partitionName) + : latestOffsetsByPartition.get(partitionName); if (offsets == null) { - throw new IllegalStateException("no offsets programmed for partition '" + partitionName + "'"); + throw new IllegalStateException("no " + (earliest ? "earliest" : "latest") + + " offsets programmed for partition '" + partitionName + "'"); } return offsets; } diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java index e4bacb68b9fdf3..ac98e23ca7e875 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java @@ -156,6 +156,37 @@ public String toString() { static final class LakeRange implements ConnectorScanRange { private static final long serialVersionUID = 1L; + /** What the paimon connector calls the bucket a split holds; the cross-connector contract. */ + static final String BUCKET_PROPERTY = "paimon.bucket"; + + private final Map properties; + private final Map partitionValues; + + LakeRange() { + this(Collections.emptyMap(), Collections.emptyMap()); + } + + private LakeRange(Map properties, Map partitionValues) { + this.properties = properties; + this.partitionValues = partitionValues; + } + + /** A split of {@code bucket} of an unpartitioned table. */ + static LakeRange inBucket(int bucket) { + return inBucket(bucket, Collections.emptyMap()); + } + + /** A split of {@code bucket} of the partition with these values. */ + static LakeRange inBucket(int bucket, Map partitionValues) { + return new LakeRange( + Collections.singletonMap(BUCKET_PROPERTY, String.valueOf(bucket)), partitionValues); + } + + /** A split that does not say which bucket it holds — what an older paimon plugin produces. */ + static LakeRange withoutABucket() { + return new LakeRange(); + } + @Override public String getTableFormatType() { return "paimon"; @@ -163,7 +194,17 @@ public String getTableFormatType() { @Override public Map getProperties() { - return Collections.emptyMap(); + return properties; + } + + @Override + public Map getPartitionValues() { + return partitionValues; + } + + @Override + public String toString() { + return "LakeRange" + properties + partitionValues; } } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 6dd1849cc5a9ce..87331232219f0f 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -478,11 +478,13 @@ struct TTableFormatFileDesc { // ES per-shard parameters (used when table_format_type == "es") // Contains: index, type, shard_id, host_port, es_hosts 13: optional map es_params - // Fluss per-split parameters (used when table_format_type == "fluss"). - // Carries ONLY what varies per split: partition/bucket identity, range type, log offsets, - // kv/lake snapshot ids and lake splits. Everything constant for the whole scan (bootstrap - // servers, table identity, client/table options) lives in TFileScanRangeParams.fluss_properties - // so it is not re-serialized once per bucket. + // Fluss per-split parameters (used when table_format_type is "fluss" or "fluss_union"). + // Carries ONLY what varies per split: partition/bucket identity, range type, log offsets and + // the kv snapshot id; a "fluss_union" split is a lake split of another connector wrapped in a + // description of the log tail that suppresses its rows, so it carries that tail here and keeps + // the wrapped connector's own params untouched. Everything constant for the whole scan + // (bootstrap servers, table identity, client/table options) lives in + // TFileScanRangeParams.fluss_properties so it is not re-serialized once per bucket. // Untyped on purpose: BE C++ holds no fluss logic, it hands this map straight to the Java // scanner, so a typed struct would only add a transcription step (see es_params, jdbc_params). 14: optional map fluss_params From 846c50affde6d2565451192786d6dd27f3482fc6 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 21:07:38 +0800 Subject: [PATCH 30/35] [feat](fluss) Read the log tail of a primary-key table as the state it ended in A primary-key table read as its lake plus its log tail needs the fluss half of that read: the change log after the offset the lake was tiered at, replayed by key. Returning that log as it stands returns every intermediate state, so a key written three times comes back three times. The three stopping rules of a bounded log range now live in one place (BoundedLogRecords) instead of two: the log reader drops the change type and takes the row, the tail reader replays the change types by key. The rule they exist for -- a fetch that consumed up to the stop without yielding a record there -- only ever shows itself as a query that never returns, which is exactly the kind of thing a second copy gets wrong. What the tail owes is narrower than a whole-bucket read, and every test turns on it: a key nobody touched in the tail must NOT come back (the lake has it), a key updated must come back once with the later value, a key deleted must come back not at all. A delete leaves a tombstone rather than removing the key -- BE's C++ side hides the lake rows for every key this range touched, deleted ones included, so the tombstone is what makes "this row disappeared on purpose" countable instead of indistinguishable from a key never written. Replay needs the primary key whether or not the query selects it, so the reader appends the missing key columns to what it asks fluss for and keeps the requested columns at the front; the row loop reads those positionally and never learns anything was added. The tail is held in memory in a process shared by every query on this BE, so fluss.union_read.max_tail_rows bounds it and a tail that outgrew it is refused rather than read. 12 tests against a real cluster, 0 skipped; the 24 existing log and primary-key tests stay green through the refactor. Mutation run 7 of 8 red: ignoring deletes, removing the key instead of tombstoning it, reading from the start of the log, never reaching the ceiling, dropping the primary-key guard, accepting an empty range, and putting the key columns anywhere but the front. The eighth -- admitting a record AT the stopping offset -- survives because it is unreachable over a contiguous log: the rule below it stops at stop-1 first, so only a gap (control records) can reach it and a client cannot produce one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../doris/fluss/BoundedLogBatchScanner.java | 84 +-- .../apache/doris/fluss/BoundedLogRecords.java | 144 +++++ .../apache/doris/fluss/FlussJniScanner.java | 85 ++- .../doris/fluss/PkTailBatchScanner.java | 238 ++++++++ .../doris/fluss/FlussJniScannerLogTest.java | 12 +- .../fluss/FlussJniScannerPkTailTest.java | 559 ++++++++++++++++++ 6 files changed, 1030 insertions(+), 92 deletions(-) create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java create mode 100644 fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java create mode 100644 fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java index 48d94cc6e842f6..aac1825b589343 100644 --- a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java @@ -20,8 +20,6 @@ import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.scanner.ScanRecord; import org.apache.fluss.client.table.scanner.batch.BatchScanner; -import org.apache.fluss.client.table.scanner.log.LogScanner; -import org.apache.fluss.client.table.scanner.log.ScanRecords; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.row.InternalRow; import org.apache.fluss.utils.CloseableIterator; @@ -34,30 +32,16 @@ /** * One bucket of a fluss log table over {@code [logStartOffset, logStopOffset)}, as a bounded scanner. * - *

A fluss log scanner is a streaming reader with no end, so the bound has to be imposed here, and - * reaching it has to be detected three ways — all of them taken from fluss's own bounded reader, - * {@code KvSnapshotAndLogBatchScanner#pollLogRecords}: - *

    - *
  • a record at or past the stopping offset is not ours — drop it and stop;
    • - *
  • after the record at {@code stopping - 1}, stop immediately. The record AT the stopping offset - * may never exist (it is where the log had got to, not a row), so polling on would block - * forever;
    • - *
  • if the fetch consumed up to the stopping offset without yielding a record there, stop too. - * That is the case the first two miss: the tail of the range can be control records, which - * occupy offsets but are never handed to a scanner.
    • - *
+ *

Every record of a log table is a row of it, so this is {@link BoundedLogRecords} with the change + * type dropped — an append-only log has nothing to replay. * - *

Shaped as a {@link BatchScanner} so that the log read and the primary-key read - * ({@code KvSnapshotAndLogBatchScanner}, which already is one) present the same interface to - * {@link FlussJniScanner} — the difference between the two belongs here, not in the row loop. + *

Shaped as a {@link BatchScanner} so that the log read and the primary-key reads + * ({@code KvSnapshotAndLogBatchScanner} and {@link PkTailBatchScanner}) present the same interface to + * {@link FlussJniScanner} — what separates them belongs here, not in the row loop. */ class BoundedLogBatchScanner implements BatchScanner { - private final LogScanner logScanner; - private final TableBucket tableBucket; - private final long logStopOffset; - - private boolean finished; + private final BoundedLogRecords records; /** * @param projection table field indexes to read, in the order the caller wants them back; must @@ -66,67 +50,25 @@ class BoundedLogBatchScanner implements BatchScanner { */ BoundedLogBatchScanner(Table table, TableBucket tableBucket, int[] projection, long logStartOffset, long logStopOffset) { - this.tableBucket = tableBucket; - this.logStopOffset = logStopOffset; - LogScanner scanner = table.newScan().project(projection).createLogScanner(); - try { - Long partitionId = tableBucket.getPartitionId(); - if (partitionId == null) { - scanner.subscribe(tableBucket.getBucket(), logStartOffset); - } else { - scanner.subscribe(partitionId, tableBucket.getBucket(), logStartOffset); - } - } catch (RuntimeException | Error e) { - // The scanner is already running its fetcher threads; leaving it unreferenced would keep - // them alive for the life of the BE process. - try { - scanner.close(); - } catch (Exception closeFailure) { - e.addSuppressed(closeFailure); - } - throw e; - } - this.logScanner = scanner; + this.records = new BoundedLogRecords(table, tableBucket, projection, + logStartOffset, logStopOffset); } @Override public CloseableIterator pollBatch(Duration timeout) { - if (finished) { + if (records.isFinished()) { return null; } - ScanRecords scanRecords = logScanner.poll(timeout); - List rows = new ArrayList<>(); - for (ScanRecord record : scanRecords.records(tableBucket)) { - long offset = record.logOffset(); - if (offset >= logStopOffset) { - // Past the end of this range: another query's rows, not ours. - finished = true; - break; - } + List batch = records.poll(timeout); + List rows = new ArrayList<>(batch.size()); + for (ScanRecord record : batch) { rows.add(record.getRow()); - if (offset >= logStopOffset - 1) { - // The last record of the range. Do not poll again: the record AT the stopping offset - // may not exist, and waiting for it never returns. - finished = true; - break; - } - } - Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); - if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { - // The fetch reached the end of the range without necessarily yielding a record there — the - // tail can be control records, which take offsets but are never scanned. Without this the - // loop would poll for a row that is never coming. - finished = true; } return CloseableIterator.wrap(rows.iterator()); } @Override public void close() throws IOException { - try { - logScanner.close(); - } catch (Exception e) { - throw new IOException("Failed to close the fluss log scanner", e); - } + records.close(); } } diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java new file mode 100644 index 00000000000000..9db6bb3f11e4b1 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.TableBucket; + +import java.io.Closeable; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * One bucket's log over {@code [logStartOffset, logStopOffset)}, polled in batches. + * + *

A fluss log scanner is a streaming reader with no end, so the bound has to be imposed here, and + * reaching it has to be detected three ways — all of them taken from fluss's own bounded reader, + * {@code KvSnapshotAndLogBatchScanner#pollLogRecords}: + *

    + *
  • a record at or past the stopping offset is not ours — drop it and stop;
    • + *
  • after the record at {@code stopping - 1}, stop immediately. The record AT the stopping offset + * may never exist (it is where the log had got to, not a row), so polling on would block + * forever;
    • + *
  • if the fetch consumed up to the stopping offset without yielding a record there, stop too. + * That is the case the first two miss: the tail of the range can be control records, which + * occupy offsets but are never handed to a scanner.
    • + *
+ * + *

This is the only place those three live. Both readers of a bounded log range consume it — + * {@link BoundedLogBatchScanner}, which wants the rows and throws the change type away, and + * {@link PkTailBatchScanner}, which replays the change types by key. A second copy of the stopping + * rules would be a second chance to get one of them wrong, and the one they exist for (the third) + * only shows itself as a query that never returns. + * + *

Records are handed out as fluss returned them, and the caller may hold them past the next poll: + * that is what fluss's own bounded primary-key reader does while it collects a whole range before + * merging it. + */ +class BoundedLogRecords implements Closeable { + + private final LogScanner logScanner; + private final TableBucket tableBucket; + private final long logStopOffset; + + private boolean finished; + + /** + * @param projection table field indexes to read, in the order the caller wants them back; must + * not be empty, which fluss rejects outright + * @param logStartOffset a real offset, or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel + */ + BoundedLogRecords(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset) { + this.tableBucket = tableBucket; + this.logStopOffset = logStopOffset; + LogScanner scanner = table.newScan().project(projection).createLogScanner(); + try { + Long partitionId = tableBucket.getPartitionId(); + if (partitionId == null) { + scanner.subscribe(tableBucket.getBucket(), logStartOffset); + } else { + scanner.subscribe(partitionId, tableBucket.getBucket(), logStartOffset); + } + } catch (RuntimeException | Error e) { + // The scanner is already running its fetcher threads; leaving it unreferenced would keep + // them alive for the life of the BE process. + try { + scanner.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + this.logScanner = scanner; + } + + /** Whether the range has been read to its end; polling after that returns nothing more. */ + boolean isFinished() { + return finished; + } + + /** + * The next records of the range, in log order. May be empty while the fetch is still on its way, + * which says nothing about whether the range is done — {@link #isFinished()} does. + */ + List poll(Duration timeout) { + if (finished) { + return new ArrayList<>(); + } + ScanRecords scanRecords = logScanner.poll(timeout); + List records = new ArrayList<>(); + for (ScanRecord record : scanRecords.records(tableBucket)) { + long offset = record.logOffset(); + if (offset >= logStopOffset) { + // Past the end of this range: another query's rows, not ours. + finished = true; + break; + } + records.add(record); + if (offset >= logStopOffset - 1) { + // The last record of the range. Do not poll again: the record AT the stopping offset + // may not exist, and waiting for it never returns. + finished = true; + break; + } + } + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { + // The fetch reached the end of the range without necessarily yielding a record there — the + // tail can be control records, which take offsets but are never scanned. Without this the + // loop would poll for a row that is never coming. + finished = true; + } + return records; + } + + @Override + public void close() throws IOException { + try { + logScanner.close(); + } catch (Exception e) { + throw new IOException("Failed to close the fluss log scanner", e); + } + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java index a34d09439f8cf0..f4bf6d14931d11 100644 --- a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java @@ -51,12 +51,14 @@ * so every one this class needs is read through {@link #required}, which names the missing key rather * than letting a null reach fluss. * - *

Two ways to read, one loop. A log table's range is its bucket's records over + *

Three ways to read, one loop. A log table's range is its bucket's records over * {@code [start, stop)}, in log order. A primary-key table's range cannot be read that way — its log * is a change log, so replaying it verbatim returns superseded and deleted rows — and is read instead * as a kv snapshot merged with the change log that followed it, by fluss's own - * {@code KvSnapshotAndLogBatchScanner}. Both are fluss {@code BatchScanner}s, so the row loop below - * does not know which one it is draining. + * {@code KvSnapshotAndLogBatchScanner}. A primary-key table whose rows are mostly in its lake needs + * only the tail of that change log, replayed by key into the state it ended in + * ({@link PkTailBatchScanner}); the lake half of that read never comes through here. All three are + * fluss {@code BatchScanner}s, so the row loop below does not know which one it is draining. * *

Partition columns are not read here. FE declares them to the engine, which leaves them out of * {@code required_fields} and fills them from the range itself, so the projection this builds covers @@ -77,9 +79,12 @@ public class FlussJniScanner extends JniScanner { private static final String LOG_START_OFFSET = "fluss.log_start_offset"; private static final String LOG_STOP_OFFSET = "fluss.log_stop_offset"; private static final String KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; + /** Scan-level, and only when a primary-key table is read as its lake plus this tail. */ + private static final String MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; private static final String RANGE_TYPE_LOG = "LOG"; private static final String RANGE_TYPE_PK_FULL = "PK_FULL"; + private static final String RANGE_TYPE_PK_TAIL = "PK_TAIL"; /** * How long one poll waits for data. Only affects how often the loop spins, never correctness: the @@ -91,11 +96,13 @@ public class FlussJniScanner extends JniScanner { private final ClassLoader classLoader; private final FlussColumnValue columnValue; - private final boolean primaryKeyRange; + private final String rangeType; private final long logStopOffset; private final long logStartOffset; /** Kv snapshot to start a primary-key read from; {@code -1} when the bucket has never had one. */ private final long kvSnapshotId; + /** How many change log records a {@code PK_TAIL} range may hold while replaying it. */ + private final long maxTailRows; private final int bucketId; /** {@code null} on an unpartitioned table, which fluss subscribes to by bucket alone. */ private final Long partitionId; @@ -103,6 +110,8 @@ public class FlussJniScanner extends JniScanner { private Connection connection; private Table table; private BatchScanner scanner; + /** The same object as {@link #scanner} on a {@code PK_TAIL} range, for what it counted. */ + private PkTailBatchScanner tailScanner; /** Fluss types of the projected columns, positionally aligned with {@link #fields}. */ private List projectedTypes; @@ -115,20 +124,31 @@ public FlussJniScanner(int batchSize, Map params) { this.params = params; this.classLoader = this.getClass().getClassLoader(); - String rangeType = required(RANGE_TYPE); - this.primaryKeyRange = RANGE_TYPE_PK_FULL.equals(rangeType); - if (!primaryKeyRange && !RANGE_TYPE_LOG.equals(rangeType)) { - // Union ranges are planned but not read yet; failing here beats returning the fluss half - // of a union read as if it were the whole table. + this.rangeType = required(RANGE_TYPE); + if (!RANGE_TYPE_LOG.equals(rangeType) && !RANGE_TYPE_PK_FULL.equals(rangeType) + && !RANGE_TYPE_PK_TAIL.equals(rangeType)) { + // The suppression descriptor a union read attaches to a lake split also travels as a range + // type, and is read by BE's C++ side alone; one reaching here would mean a lake split was + // handed to the fluss scanner, and reading it as a fluss range would return nothing at all. throw new IllegalArgumentException( - "fluss scan range type '" + rangeType + "' is not supported yet; expected " - + RANGE_TYPE_LOG + " or " + RANGE_TYPE_PK_FULL); + "fluss scan range type '" + rangeType + "' is not one this scanner reads; expected " + + RANGE_TYPE_LOG + ", " + RANGE_TYPE_PK_FULL + " or " + RANGE_TYPE_PK_TAIL); } // Every range field is parsed here, before open() creates a connection: a range that cannot be // read should say so without having contacted the cluster first. this.logStopOffset = Long.parseLong(required(LOG_STOP_OFFSET)); this.logStartOffset = Long.parseLong(required(LOG_START_OFFSET)); - this.kvSnapshotId = primaryKeyRange ? Long.parseLong(required(KV_SNAPSHOT_ID)) : -1L; + this.kvSnapshotId = RANGE_TYPE_PK_FULL.equals(rangeType) + ? Long.parseLong(required(KV_SNAPSHOT_ID)) : -1L; + this.maxTailRows = RANGE_TYPE_PK_TAIL.equals(rangeType) ? maxTailRows() : 0L; + if (RANGE_TYPE_PK_TAIL.equals(rangeType) && logStartOffset >= logStopOffset) { + // A bucket whose lake has caught up with its log contributes nothing, and planning says so + // by not planning a range at all. One arriving here means the lake half and the fluss half + // were bounded by different offsets, which is a duplicate or a missing row either way. + throw new IllegalArgumentException("a fluss log tail must read something, but bucket " + + required(BUCKET_ID) + " was given [" + logStartOffset + ", " + logStopOffset + + ")"); + } this.bucketId = Integer.parseInt(required(BUCKET_ID)); String partition = params.get(PARTITION_ID); this.partitionId = partition == null ? null : Long.parseLong(partition); @@ -170,10 +190,7 @@ public void open() throws IOException { TableBucket tableBucket = partitionId == null ? new TableBucket(tableId, bucketId) : new TableBucket(tableId, partitionId, bucketId); - scanner = primaryKeyRange - ? primaryKeyScanner(tableBucket, projection) - : new BoundedLogBatchScanner(table, tableBucket, scanProjection(projection), - logStartOffset, logStopOffset); + scanner = createScanner(tableBucket, projection); } catch (Throwable e) { try { close(); @@ -188,6 +205,24 @@ public void open() throws IOException { } } + /** + * The reader for this kind of range. All three are fluss {@code BatchScanner}s, which is what lets + * {@link #getNext} drain any of them without knowing which it has. + */ + private BatchScanner createScanner(TableBucket tableBucket, int[] projection) { + switch (rangeType) { + case RANGE_TYPE_PK_FULL: + return primaryKeyScanner(tableBucket, projection); + case RANGE_TYPE_PK_TAIL: + tailScanner = new PkTailBatchScanner(table, tableBucket, projection, + logStartOffset, logStopOffset, maxTailRows); + return tailScanner; + default: + return new BoundedLogBatchScanner(table, tableBucket, scanProjection(projection), + logStartOffset, logStopOffset); + } + } + /** * The kv snapshot of this bucket merged with the change log that followed it, by fluss's own * bounded primary-key reader. That class is {@code @Internal} — this connector is pinned to the @@ -318,9 +353,27 @@ public Map getStatistics() { Map statistics = new HashMap<>(); statistics.put("counter:FlussJniRowsRead", String.valueOf(rowsRead)); statistics.put("gauge:FlussJniRequiredFieldCount", String.valueOf(fields.length)); + if (tailScanner != null) { + // What the tail cost and what it hid: the records replayed, and the keys it ended deleted — + // those are lake rows that disappear with nothing returned in their place. + statistics.put("counter:FlussJniTailRecordsRead", + String.valueOf(tailScanner.getRecordsRead())); + statistics.put("counter:FlussJniTailTombstoneKeys", + String.valueOf(tailScanner.getTombstoneKeys())); + } return statistics; } + /** The tail ceiling, which planning sends with every {@code PK_TAIL} range of a union read. */ + private long maxTailRows() { + long rows = Long.parseLong(required(MAX_TAIL_ROWS)); + if (rows <= 0) { + throw new IllegalArgumentException("fluss scanner parameter '" + MAX_TAIL_ROWS + + "' is " + rows + "; expected a positive number of rows"); + } + return rows; + } + private String required(String key) { String value = params.get(key); if (value == null) { diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java new file mode 100644 index 00000000000000..7bdd80d7590e56 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KeyEncoder; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.CloseableIterator; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The log tail of one bucket of a primary-key table, returned as the state that tail ended in. + * + *

This is the fluss half of a union read: the lake holds the bucket up to the offset it was tiered + * at, and this contributes what has happened to that bucket since. The change log cannot be returned + * as it stands — it records every intermediate state, so a key written three times appears three + * times — so the range is replayed by key first, exactly the way fluss's own bounded primary-key + * reader reduces the log half of a snapshot read. + * + *

A delete leaves a tombstone rather than removing the key. The two look interchangeable + * from here, and are not: the lake rows this tail supersedes are hidden by a key set that BE's C++ + * side builds from the same offsets, and that set covers every key the tail touched — deleted ones + * included. So a key deleted in the tail is already gone from the lake half, and what this reader owes + * is only the keys that survived. Keeping the tombstone keeps that fact countable + * ({@link #getTombstoneKeys()}) instead of indistinguishable from a key that was never written. + * + *

The key is the physical primary key — the primary key minus the partition columns. Rows + * are only ever compared within one bucket of one partition, where the partition columns are equal by + * construction, and both halves of the union agree on that: fluss's own kv storage keys the same way. + * + *

The whole range is collected before anything is returned, which is what a merge of any shape + * costs; {@code fluss.union.max_tail_rows} is what stops a tail that grew unbounded (tiering stopped, + * say) from being read into a shared BE process without a limit. + * + *

Rows are held across polls, as fluss's own bounded reader does with the same records. + */ +class PkTailBatchScanner implements BatchScanner { + + private final BoundedLogRecords records; + private final TableBucket tableBucket; + private final long maxTailRows; + + /** Reads the key columns out of a scan row; reused, because each key is encoded on the spot. */ + private final ProjectedRow keyRow; + private final KeyEncoder keyEncoder; + + /** + * The replayed state: the last record of each key, in the order the key was first seen. A + * {@code null} value is a tombstone — the key ended the range deleted. + */ + private final Map state = new LinkedHashMap<>(); + + private long recordsRead; + private long tombstoneKeys; + private boolean emitted; + + /** + * @param projection table field indexes the query asked for, in the order it wants them back; + * may be empty, which a count-shaped query produces + * @param logStartOffset where the lake snapshot this tail follows left off + * @param maxTailRows how many change log records this range may hold before it is refused + */ + PkTailBatchScanner(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset, long maxTailRows) { + TableInfo tableInfo = table.getTableInfo(); + if (!tableInfo.hasPrimaryKey()) { + // Replaying by key needs a key. A log table's records are rows in their own right and are + // read as a LOG range; reaching here with one means the range was planned as the wrong kind. + throw new IllegalArgumentException("a primary-key log tail was planned for " + + tableInfo.getTablePath() + ", which has no primary key"); + } + this.tableBucket = tableBucket; + this.maxTailRows = maxTailRows; + + List primaryKeys = tableInfo.getPhysicalPrimaryKeys(); + RowType rowType = tableInfo.getRowType(); + // What to ask fluss for: the requested columns, plus any key column not among them. The key is + // needed to replay whether or not the query selects it, and appending keeps the requested + // columns at the front — so a returned row's first fields are the requested ones, in order, and + // the row loop reads them positionally without knowing this reader added anything. + List scanProjection = new ArrayList<>(projection.length + primaryKeys.size()); + for (int index : projection) { + scanProjection.add(index); + } + int[] keyIndexesInTable = new int[primaryKeys.size()]; + int[] keyIndexesInScanRow = new int[primaryKeys.size()]; + for (int i = 0; i < primaryKeys.size(); i++) { + int indexInTable = rowType.getFieldIndex(primaryKeys.get(i)); + keyIndexesInTable[i] = indexInTable; + int indexInScanRow = scanProjection.indexOf(indexInTable); + if (indexInScanRow < 0) { + scanProjection.add(indexInTable); + indexInScanRow = scanProjection.size() - 1; + } + keyIndexesInScanRow[i] = indexInScanRow; + } + this.keyRow = ProjectedRow.from(keyIndexesInScanRow); + // The table's own key encoder, chosen the way fluss chooses it for this table. Nothing outside + // this JVM sees these bytes — they only have to tell two different keys apart, which is the one + // thing every encoder this factory returns guarantees. + this.keyEncoder = KeyEncoder.ofPrimaryKeyEncoder( + Schema.getKeyRowType(tableInfo.getSchema(), keyIndexesInTable), + primaryKeys, tableInfo.getTableConfig(), tableInfo.isDefaultBucketKey()); + + int[] fields = new int[scanProjection.size()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = scanProjection.get(i); + } + this.records = new BoundedLogRecords(table, tableBucket, fields, + logStartOffset, logStopOffset); + } + + @Override + public CloseableIterator pollBatch(Duration timeout) throws IOException { + if (emitted) { + return null; + } + if (!records.isFinished()) { + for (ScanRecord record : records.poll(timeout)) { + replay(record); + } + // An empty batch, not the end: the caller polls again. Only when the range has been read to + // its end is there an answer to give. + return CloseableIterator.emptyIterator(); + } + emitted = true; + List surviving = new ArrayList<>(); + for (InternalRow row : state.values()) { + if (row != null) { + surviving.add(row); + } + } + tombstoneKeys = state.size() - surviving.size(); + // The surviving rows are held by the list now; the keys that indexed them are the tail's whole + // memory cost and nothing reads them again. + state.clear(); + return CloseableIterator.wrap(surviving.iterator()); + } + + private void replay(ScanRecord record) { + recordsRead++; + if (recordsRead > maxTailRows) { + throw new IllegalStateException("the log tail of bucket " + tableBucket.getBucket() + + " holds more than " + maxTailRows + " change log records, the limit set by '" + + "fluss.union_read.max_tail_rows'. Read the table as pure fluss with" + + " 'fluss.union_read.mode=disabled', wait for tiering to move the tail into the" + + " lake, or raise the limit"); + } + InternalRow row = record.getRow(); + TailKey key = new TailKey(keyEncoder.encodeKey(keyRow.replaceRow(row))); + switch (record.getChangeType()) { + case INSERT: + case UPDATE_AFTER: + state.put(key, row); + break; + case DELETE: + case UPDATE_BEFORE: + state.put(key, null); + break; + default: + // A primary-key table's change log holds no other kind. One appearing means fluss now + // describes a change this reader has never replayed, and guessing at it would return a + // row nobody can check. + throw new IllegalStateException("the change log of a primary-key table produced a '" + + record.getChangeType() + "' record, which this reader does not know how to" + + " replay"); + } + } + + /** How many change log records the tail held. */ + long getRecordsRead() { + return recordsRead; + } + + /** + * How many keys the tail ended deleted, known once the range has been replayed. Those are the keys + * whose lake rows disappear without anything being returned in their place. + */ + long getTombstoneKeys() { + return tombstoneKeys; + } + + @Override + public void close() throws IOException { + records.close(); + } + + /** An encoded primary key, by value: {@code byte[]} is compared by identity on its own. */ + private static final class TailKey { + + private final byte[] encoded; + private final int hash; + + TailKey(byte[] encoded) { + this.encoded = encoded; + this.hash = Arrays.hashCode(encoded); + } + + @Override + public boolean equals(Object other) { + return other instanceof TailKey && Arrays.equals(encoded, ((TailKey) other).encoded); + } + + @Override + public int hashCode() { + return hash; + } + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java index 873b851c167dba..f90f2a88cacf02 100644 --- a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java @@ -443,17 +443,19 @@ public void missingParameterIsNamed() { } /** - * Union ranges are planned by FE but not readable here yet. Refusing beats reading their fluss - * half and returning it as if it were the whole table. + * A union read of a primary-key table puts a fourth range type on the wire — the suppression + * descriptor that rides along with a lake split — and BE's C++ side is the only reader of it. One + * arriving here means a lake split was handed to this scanner, which would return no rows at all + * for a split that holds many. */ @Test - public void unsupportedRangeTypeIsRefused() { + public void rangeTypeThisScannerDoesNotReadIsRefused() { Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); - params.put("fluss.range_type", "UNION_PK"); + params.put("fluss.range_type", "LAKE_SUPPRESS"); IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> new FlussJniScanner(1024, params)); - Assertions.assertTrue(e.getMessage().contains("UNION_PK"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("LAKE_SUPPRESS"), e.getMessage()); } /** A column dropped between planning and reading must say so, not read a neighbouring column. */ diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java new file mode 100644 index 00000000000000..7b5c5e07256c00 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java @@ -0,0 +1,559 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Reading the log tail of a primary-key table — the fluss half of a union read — against a real + * cluster in this JVM. + * + *

What this range means is narrower than the whole-bucket read next door, and the difference is the + * point of every test here: the lake holds the bucket as it stood at the offset it was tiered at, so + * this range owes exactly what has happened since, replayed into the state it ended in. A key nobody + * touched in the tail must NOT come back — the lake already has it, and returning it again is a + * duplicate row. A key updated in the tail must come back once, with the final value. A key deleted in + * the tail must come back not at all, and that is not the same as never having been written: the lake + * row it supersedes is hidden by a key set BE's C++ side builds from these same offsets, so the row + * disappearing is the whole intent. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerPkTailTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands in the bucket the test reads the tail of. */ + private static final int BUCKETS = 1; + + /** A ceiling no fixture here comes near; the one test about the ceiling sets its own. */ + private static final long NO_CEILING_IN_PRACTICE = 1_000_000L; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_tail_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- what the tail owes + + /** + * The three things at once, on one tail: a key the tail updated comes back once with the later + * value, a key the tail deleted does not come back, a key the tail inserted comes back — and the + * key nobody touched does not come back at all, because it is the lake's to return. + * + *

Returning the change log as it stands would give five rows for these three keys. + */ + @Test + public void tailComesBackAsTheStateItEndedIn() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_mixed"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(4, "four")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals( + new Object[][] {{1, "one-updated"}, {4, "four"}}, sortById(rows)); + } + + /** + * A key written only before the tail belongs to the lake half. Reading from the beginning of the + * log instead of from where the lake left off returns it here too, and the query then has it twice. + */ + @Test + public void rowsBeforeTheTailAreLeftToTheLake() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_start"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(3, "three")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals(new Object[][] {{3, "three"}}, sortById(rows)); + } + + /** + * The tail ends where planning said it ends. A write that lands after that belongs to the next + * query — and here it would also disagree with the key set the lake half was filtered by, which was + * built from the same two offsets. + */ + @Test + public void writesAfterTheStoppingOffsetAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_late_write"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(2, "two")); + long stop = latestOffset(tablePath); + upsert(tablePath, row(3, "three")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, stop, NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals(new Object[][] {{2, "two"}}, sortById(rows)); + } + + /** + * A tail that only deletes returns nothing, and says so: the deleted key is counted, because the + * lake row it hides is dropped on the strength of this range having seen the key. A tail that + * quietly returned no rows and counted nothing would look exactly like a tail that read nothing at + * all. + * + *

The record count is asserted too — an update of an existing key is logged as a pair, a delete + * as one record. A fluss upgrade that changes the shape of the change log should land here. + */ + @Test + public void keysDeletedInTheTailAreCountedRatherThanReturned() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_delete"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + Map statistics = new HashMap<>(); + Object[][] rows = read(params, 1024, statistics); + + Assertions.assertArrayEquals(new Object[][] {{1, "one-updated"}}, sortById(rows)); + Assertions.assertEquals("1", statistics.get("counter:FlussJniTailTombstoneKeys")); + Assertions.assertEquals("3", statistics.get("counter:FlussJniTailRecordsRead"), + "an update is two change log records and a delete is one"); + } + + // ---------------------------------------------------------------- projection and shape + + /** + * Replaying needs the primary key whether or not the query selects it, so the reader asks fluss for + * it and does not return it. Getting that wrong shows up as an extra column — or, if the key were + * simply not fetched, as two rows for one key. + */ + @Test + public void projectionExcludingThePrimaryKeyKeepsTheRequestedShape() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("a", DataTypes.INT()) + .column("b", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + upsert(tablePath, GenericRow.of(1, 10, BinaryString.fromString("x"))); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, GenericRow.of(1, 11, BinaryString.fromString("y"))); + upsert(tablePath, GenericRow.of(1, 12, BinaryString.fromString("z"))); + + Object[][] rows = read(tailParams(tablePath, columns("b", "string", "a", "int"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {"z", 12}, rows[0]); + } + + /** + * A query can need no column of this scanner at all — {@code select dt, count(*) ... group by dt} + * over a partitioned table — and the count still has to be of the replayed rows. Counting change + * log records instead reports four where the tail contributes two. + */ + @Test + public void projectingNoColumnStillCountsTheReplayedRows() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_count_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(3, "three")); + + Map params = tailParams(tablePath, columns(), lakeEnd, + latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + int rows = 0; + FlussJniScanner scanner = new FlussJniScanner(1024, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + rows += scanner.getTable().getNumRows(); + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + + Assertions.assertEquals(2, rows); + } + + /** More surviving rows than fit in one batch: the reader has to resume where it stopped. */ + @Test + public void replayedRowsSpanSeveralBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_batches"); + createPkTable(tablePath); + upsert(tablePath, row(0, "v0")); + long lakeEnd = latestOffset(tablePath); + GenericRow[] tail = new GenericRow[50]; + for (int i = 1; i <= 50; i++) { + tail[i - 1] = row(i, "v" + i); + } + upsert(tablePath, tail); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 7); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(50, sorted.length); + for (int i = 1; i <= 50; i++) { + Assertions.assertArrayEquals(new Object[] {i, "v" + i}, sorted[i - 1], "row " + i); + } + } + + /** + * A partitioned table's tail is a tail of one partition's bucket. Subscribing without the partition + * reads a different tablet altogether, and the lake half it is joined to is this partition's. + */ + @Test + public void partitionedTailIsReadThroughItsPartitionBucket() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_part"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), BinaryString.fromString("a")), + GenericRow.of(2, BinaryString.fromString("2026_08_04"), BinaryString.fromString("b"))); + long lakeEnd = partitionOffset(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), + BinaryString.fromString("a-updated")), + GenericRow.of(3, BinaryString.fromString("2026_08_04"), + BinaryString.fromString("c"))); + + // The partition column is not read here: FE declares it and BE materializes it from the range. + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, partitionOffset(tablePath, "2026_08_03"), NO_CEILING_IN_PRACTICE); + params.put("fluss.partition_id", String.valueOf(partitionId)); + + Object[][] rows = read(params, 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "a-updated"}}, sortById(rows)); + } + + // ---------------------------------------------------------------- what must fail loudly + + /** + * The tail is held in memory to be replayed, in a process shared by every query on this BE. A tail + * that outgrew its ceiling — tiering stopped, say — has to say so rather than read on. + */ + @Test + public void tailBiggerThanItsCeilingIsRefused() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_ceiling"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(2, "two"), row(3, "three"), row(4, "four")); + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), 2L); + + Exception failure = Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + Assertions.assertTrue(messageChain(failure).contains("fluss.union_read.max_tail_rows"), + "the ceiling that was hit should be named, but the failure was: " + + messageChain(failure)); + } + + /** + * An empty tail is not something planning produces — a bucket whose lake has caught up contributes + * no range at all. One arriving means the two halves were bounded by different offsets, which is a + * duplicated or a missing row either way, and reading nothing would hide it. + */ + @Test + public void emptyTailRangeIsRefused() { + Map params = tailParams(TablePath.of(db, "tail_empty"), + columns("id", "int"), 7L, 7L, NO_CEILING_IN_PRACTICE); + + Exception failure = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(failure.getMessage().contains("must read something"), failure.getMessage()); + } + + /** + * A log table has no key to replay by, and its records are rows in their own right. A tail range + * pointed at one means planning produced the wrong kind of range; replaying it by "the key" would + * mean inventing one. + */ + @Test + public void tailOfALogTableIsRefused() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_log_table"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + writer.append(row(1, "one")); + writer.flush(); + } + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + 0L, latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + + Exception failure = Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + Assertions.assertTrue(messageChain(failure).contains("no primary key"), + "the failure should name what is missing, but it was: " + messageChain(failure)); + } + + /** A ceiling that cannot bound anything is a configuration mistake, not a tail of size zero. */ + @Test + public void nonPositiveCeilingIsRefused() { + Map params = tailParams(TablePath.of(db, "tail_bad_ceiling"), + columns("id", "int"), 0L, 7L, 0L); + + Exception failure = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(failure.getMessage().contains("fluss.union.max_tail_rows"), + failure.getMessage()); + } + + // ---------------------------------------------------------------- helpers + + private void createPkTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + } + + private static GenericRow row(int id, String name) { + return GenericRow.of(id, BinaryString.fromString(name)); + } + + private static void upsert(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (GenericRow row : rows) { + writer.upsert(row); + } + writer.flush(); + } + } + + private static void delete(TablePath tablePath, GenericRow row) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + writer.delete(row); + writer.flush(); + } + } + + /** Where bucket 0's log has got to — what planning uses to bound a tail. */ + private static long latestOffset(TablePath tablePath) throws Exception { + return admin.listOffsets(tablePath, Collections.singletonList(0), new OffsetSpec.LatestSpec()) + .all().get().get(0); + } + + private static long partitionOffset(TablePath tablePath, String partitionName) throws Exception { + return admin.listOffsets(tablePath, partitionName, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** + * The merged map BE hands the scanner for a tail range: the range's own keys, plus the scan-level + * ceiling that only a union read of a primary-key table carries. No kv snapshot — the state before + * the tail is the lake's, not a snapshot's. + */ + private static Map tailParams(TablePath tablePath, Map columns, + long logStartOffset, long logStopOffset, long maxTailRows) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "PK_TAIL"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.log_start_offset", String.valueOf(logStartOffset)); + params.put("fluss.log_stop_offset", String.valueOf(logStopOffset)); + params.put("fluss.union.max_tail_rows", String.valueOf(maxTailRows)); + params.put("time_zone", "UTC"); + return params; + } + + private static Object[][] read(Map params, int batchSize) throws Exception { + return read(params, batchSize, new HashMap<>()); + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced, filling {@code statistics} with what the scanner counted. + * {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; reading it as + * rows would silently compare a column against a row. + */ + private static Object[][] read(Map params, int batchSize, + Map statistics) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + statistics.putAll(scanner.getStatistics()); + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } + + /** Every message down the cause chain, because open() wraps what the reader threw. */ + private static String messageChain(Throwable failure) { + StringBuilder messages = new StringBuilder(); + for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + messages.append(cause.getMessage()).append(" | "); + } + return messages.toString(); + } + + /** + * Sorted by the first column, which every fixture here makes the primary key. Replay returns rows + * in the order their keys were first seen in the tail, which is fluss's write order, not an order + * this reader promises. + */ + private static Object[][] sortById(Object[][] rows) { + Object[][] sorted = Arrays.copyOf(rows, rows.length); + Arrays.sort(sorted, Comparator.comparingInt(row -> (Integer) row[0])); + return sorted; + } +} From ecfec2a98321339cea25bc7a7dbb1c8b4232790f Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 22:02:08 +0800 Subject: [PATCH 31/35] [feat](fluss) Read a primary-key lake split without the rows its log tail replaced Planning already produces the lake half of a tiered primary-key table wrapped with the offsets of the log tail that supersedes part of it. This is the reader that honours that: it reads the tail's keys once per bucket and drops the lake rows those keys name. What the tail ended up saying about each key arrives separately, as its own range, so every row is produced exactly once and a key the tail ended by deleting is produced not at all. The lake half is read entirely by the paimon sibling's own reader stack -- every split is forwarded to it untouched, so native ORC/Parquet, serialized JNI splits, deletion vectors, schema evolution and split scheduling all stay its answer. This reader only removes rows. The filter runs on the block that comes back, in table-schema terms, rather than inside the sibling's file scan request. The key columns are ordinary projected columns there (planning keeps them in the tuple for this read), already resolved by the column mapper whichever way the lake table needs -- by name or by field id. Pushing the predicate down into the file request would save decoding the non-key columns of the suppressed rows, which is bounded by the size of the tail, and would cost a second place that resolves key columns against a file schema plus a separate implementation for the JNI child. The tail is read as a plain bounded log range, not as its surviving state: a lake row whose key the tail deleted has to disappear just as surely as one it updated, and the surviving state would not name it. One bucket's tail is one read however many of its splits this BE was given, held by the scan node's split cache under the tail's own offsets, so an unpartitioned table's bucket cannot collide with a partition's. Its size is bounded by the same limit and counted the same way as the Java half counts it -- change log records over the same offsets -- so a table over the limit is over it on both sides rather than in whichever half happened to run first. Three things fail loud instead of degrading, because each returns the superseded rows a second time and no row count reveals it: a key column the scan does not project (planning promised it would), a wrapped split with no tail bound to it, and a block read before a tail was bound. Aggregate pushdown is withheld from the lake half for the same reason: a COUNT answered from paimon's file metadata would count the rows this reader exists to remove, without ever producing a block to remove them from. 22 tests, 0 skipped; the 226 paimon, iceberg and table-reader tests around them stay green. Twelve mutations were confirmed red, including suppressing nothing, waving through a missing suppression, one cache entry for every bucket, a limit that never fires, an empty tail range, a skipped key column, forwarded aggregate pushdown, reading the tail as its surviving state, and reading the tail of a pruned split. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../table/fluss_union_lake_reader.cpp | 524 ++++++++++++++++ .../format_v2/table/fluss_union_lake_reader.h | 143 +++++ .../table/fluss_union_lake_reader_test.cpp | 593 ++++++++++++++++++ 3 files changed, 1260 insertions(+) create mode 100644 be/src/format_v2/table/fluss_union_lake_reader.cpp create mode 100644 be/src/format_v2/table/fluss_union_lake_reader.h create mode 100644 be/test/format_v2/table/fluss_union_lake_reader_test.cpp diff --git a/be/src/format_v2/table/fluss_union_lake_reader.cpp b/be/src/format_v2/table/fluss_union_lake_reader.cpp new file mode 100644 index 00000000000000..970f54c7d18b39 --- /dev/null +++ b/be/src/format_v2/table/fluss_union_lake_reader.cpp @@ -0,0 +1,524 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/fluss_union_lake_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper.h" +#include "format_v2/expr/equality_delete_predicate.h" +#include "format_v2/jni/fluss_jni_reader.h" +#include "format_v2/table/paimon_reader.h" +#include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::fluss { +namespace { + +// The scan-node properties this reader reads. The fluss connector states that `fluss.union.*` is the +// whole of what BE's C++ side knows about fluss; anything added here has to be added there too. +constexpr const char* PROP_PK_NAMES = "fluss.union.pk_names"; +constexpr const char* PROP_MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; + +// The per-range payload of a wrapped lake split. +constexpr const char* PROP_RANGE_TYPE = "fluss.range_type"; +constexpr const char* PROP_TAIL = "fluss.union.tail"; +constexpr const char* RANGE_TYPE_LAKE_SUPPRESS = "LAKE_SUPPRESS"; + +// The range this reader synthesizes to read the tail. A plain bounded log read, which is what the +// suppression set is: every key the tail touched, whatever it ended up saying about it. +constexpr const char* RANGE_TYPE_LOG = "LOG"; +constexpr const char* PROP_PARTITION_ID = "fluss.partition_id"; +constexpr const char* PROP_BUCKET_ID = "fluss.bucket_id"; +constexpr const char* PROP_LOG_START_OFFSET = "fluss.log_start_offset"; +constexpr const char* PROP_LOG_STOP_OFFSET = "fluss.log_stop_offset"; + +constexpr size_t TAIL_BATCH_ROWS = 4096; + +std::vector split_on(std::string_view value, char separator) { + std::vector parts; + size_t start = 0; + while (true) { + const auto end = value.find(separator, start); + if (end == std::string_view::npos) { + parts.push_back(value.substr(start)); + return parts; + } + parts.push_back(value.substr(start, end - start)); + start = end + 1; + } +} + +void update_counter(RuntimeProfile::Counter* counter, int64_t value) { + if (counter != nullptr) { + COUNTER_UPDATE(counter, value); + } +} + +template +bool parse_integer(std::string_view text, T* value) { + if (text.empty()) { + return false; + } + const auto result = std::from_chars(text.data(), text.data() + text.size(), *value); + return result.ec == std::errc() && result.ptr == text.data() + text.size(); +} + +} // namespace + +Status FlussUnionLakeReader::parse_tail(const std::string& spec, Tail* tail) { + DORIS_CHECK(tail != nullptr); + const auto parts = split_on(spec, ':'); + if (parts.size() != 4) { + return Status::InternalError( + "fluss union read: '{}' is not a log tail of the form " + "partitionId:bucket:start:stop", + spec); + } + // An unpartitioned table leaves the partition segment empty rather than writing a sentinel, so + // that its bucket 0 and a partitioned table's bucket 0 cannot become the same cache entry. + if (!parts[0].empty()) { + int64_t partition_id = 0; + if (!parse_integer(parts[0], &partition_id)) { + return Status::InternalError( + "fluss union read: '{}' has a partition id that is not a " + "number in log tail '{}'", + parts[0], spec); + } + } + Tail parsed; + parsed.partition_id = std::string(parts[0]); + if (!parse_integer(parts[1], &parsed.bucket_id) || + !parse_integer(parts[2], &parsed.start_offset) || + !parse_integer(parts[3], &parsed.stop_offset)) { + return Status::InternalError( + "fluss union read: log tail '{}' has a bucket or offset that is not a number", + spec); + } + if (parsed.start_offset >= parsed.stop_offset) { + // Planning never wraps a lake split whose bucket has nothing left in its log. One arriving + // here means the two halves of this read were bounded by different offsets, and that is a + // duplicated or a missing row either way. + return Status::InternalError( + "fluss union read: a suppressing log tail must contain something, but bucket {} " + "was " + "given [{}, {})", + parsed.bucket_id, parsed.start_offset, parsed.stop_offset); + } + parsed.spec = spec; + *tail = std::move(parsed); + return Status::OK(); +} + +TFileRangeDesc FlussUnionLakeReader::tail_scan_range(const Tail& tail) { + std::map params { + {PROP_RANGE_TYPE, RANGE_TYPE_LOG}, + {PROP_BUCKET_ID, std::to_string(tail.bucket_id)}, + {PROP_LOG_START_OFFSET, std::to_string(tail.start_offset)}, + {PROP_LOG_STOP_OFFSET, std::to_string(tail.stop_offset)}, + }; + if (!tail.partition_id.empty()) { + // Absent rather than -1 on an unpartitioned table: that is how the scanner tells the two + // apart, and fluss subscribes to a bucket of each by a different call. + params.emplace(PROP_PARTITION_ID, tail.partition_id); + } + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + table_format_params.__set_fluss_params(std::move(params)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_params)); + range.__set_format_type(TFileFormatType::FORMAT_JNI); + return range; +} + +Status FlussUnionLakeReader::init(format::TableReadOptions&& options) { + RETURN_IF_ERROR(format::TableReader::init(std::move(options))); + RETURN_IF_ERROR(_resolve_union_properties()); + _init_union_profile(); + + VExprContextSPtrs conjuncts; + conjuncts.reserve(_conjuncts.size()); + for (const auto& conjunct : _conjuncts) { + VExprSPtr root; + RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); + conjuncts.push_back(VExprContext::create_shared(std::move(root))); + } + _lake_reader = std::make_unique(); + RETURN_IF_ERROR(_lake_reader->init({ + .projected_columns = _projected_columns, + .conjuncts = std::move(conjuncts), + .format = _format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + .file_slot_descs = _file_slot_descs, + // Aggregate pushdown is withheld from the lake half on purpose. A COUNT answered from + // paimon's own file metadata would count the very rows this reader is about to suppress, + // and it would do so without ever producing a block to suppress them from. + .push_down_agg_type = TPushAggOp::type::NONE, + .condition_cache_digest = _condition_cache_digest, + })); + if (_batch_size > 0) { + _lake_reader->set_batch_size(_batch_size); + } + return Status::OK(); +} + +Status FlussUnionLakeReader::_resolve_union_properties() { + if (_scan_params == nullptr || !_scan_params->__isset.fluss_properties) { + return Status::InternalError( + "missing fluss_properties for a fluss union read, possibly caused by FE/BE " + "protocol " + "mismatch"); + } + const auto& properties = _scan_params->fluss_properties; + const auto names_it = properties.find(PROP_PK_NAMES); + if (names_it == properties.end() || names_it->second.empty()) { + return Status::InternalError( + "missing '{}' for a fluss union read: without the key columns the lake rows the " + "log " + "tail supersedes cannot be identified", + PROP_PK_NAMES); + } + for (const auto name : split_on(names_it->second, ',')) { + const auto column = std::ranges::find_if( + _projected_columns, + [&](const format::ColumnDefinition& candidate) { return candidate.name == name; }); + if (column == _projected_columns.end()) { + // FE keeps the key columns in the scan's tuple whenever it plans a union read, so this + // means its planning-time decision and its split-planning decision disagreed. Suppressing + // nothing would return every superseded lake row a second time, silently. + return Status::InternalError( + "fluss union read: key column '{}' is not among the columns this scan " + "projects. " + "The lake rows its log tail supersedes cannot be identified without it", + name); + } + _key_column_indexes.push_back( + cast_set(std::distance(_projected_columns.begin(), column))); + _key_columns.push_back(*column); + } + + const auto rows_it = properties.find(PROP_MAX_TAIL_ROWS); + if (rows_it == properties.end() || + !parse_integer(std::string_view(rows_it->second), &_max_tail_rows) || _max_tail_rows <= 0) { + return Status::InternalError( + "fluss union read: '{}' must be a positive number of rows, but was '{}'", + PROP_MAX_TAIL_ROWS, + rows_it == properties.end() ? std::string("missing") : rows_it->second); + } + return Status::OK(); +} + +void FlussUnionLakeReader::_init_union_profile() { + if (_scanner_profile == nullptr) { + return; + } + static const char* table_profile = file_scan_profile::TABLE_READER; + _suppressed_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionSuppressedRows", TUnit::UNIT, table_profile, 1); + _tail_keys_read_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionTailKeysRead", TUnit::UNIT, table_profile, 1); + _tail_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionTailCacheHitCount", TUnit::UNIT, table_profile, 1); +} + +Status FlussUnionLakeReader::prepare_split(const format::SplitReadOptions& options) { + DORIS_CHECK(_lake_reader != nullptr); + RETURN_IF_ERROR(_lake_reader->prepare_split(options)); + if (_lake_reader->current_split_pruned()) { + // A pruned split returns no rows, so there is nothing to suppress and no reason to spend a + // read of the tail on it. + return Status::OK(); + } + return _prepare_suppression(options); +} + +Status FlussUnionLakeReader::_prepare_suppression(const format::SplitReadOptions& options) { + const auto& range = options.current_range; + if (!range.__isset.table_format_params || !range.table_format_params.__isset.fluss_params) { + return Status::InternalError( + "missing fluss_params on a fluss union lake split, possibly caused by FE/BE " + "protocol " + "mismatch"); + } + const auto& params = range.table_format_params.fluss_params; + const auto type_it = params.find(PROP_RANGE_TYPE); + if (type_it == params.end() || type_it->second != RANGE_TYPE_LAKE_SUPPRESS) { + return Status::InternalError( + "a fluss union lake split must carry '{}={}', but carries '{}'", PROP_RANGE_TYPE, + RANGE_TYPE_LAKE_SUPPRESS, + type_it == params.end() ? std::string("nothing") : type_it->second); + } + const auto tail_it = params.find(PROP_TAIL); + if (tail_it == params.end()) { + return Status::InternalError("missing '{}' on a fluss union lake split", PROP_TAIL); + } + if (_suppression != nullptr && _suppression_tail_spec == tail_it->second) { + // Consecutive splits of one bucket are common; their suppression is the same one. + return Status::OK(); + } + Tail tail; + RETURN_IF_ERROR(parse_tail(tail_it->second, &tail)); + RETURN_IF_ERROR(_load_suppression_keys(options, tail)); + _suppression_tail_spec = tail_it->second; + return Status::OK(); +} + +Status FlussUnionLakeReader::_load_suppression_keys(const format::SplitReadOptions& options, + const Tail& tail) { + if (options.cache == nullptr) { + return Status::InternalError( + "fluss union read: no split cache to hold the keys of log tail '{}'", tail.spec); + } + // Length-prefixed so that no boundary between the fixed prefix and the tail can be reinterpreted + // as part of the tail itself. One scan node reads one table, so the tail alone identifies it. + const auto cache_key = fmt::format("fluss_union_tail:{}:{}", tail.spec.size(), tail.spec); + Status read_status = Status::OK(); + bool cache_hit = false; + auto* cached = options.cache->get( + cache_key, + [&]() -> SuppressionKeys* { + auto keys = std::make_unique(); + read_status = _read_tail_keys(tail, &keys->keys); + if (!read_status.ok()) { + return nullptr; + } + return keys.release(); + }, + &cache_hit); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(cached != nullptr); + if (cache_hit) { + update_counter(_tail_cache_hit_counter, 1); + } else { + update_counter(_tail_keys_read_counter, cast_set(cached->keys.rows())); + } + return _build_suppression_predicate(cached->keys); +} + +Block FlussUnionLakeReader::_empty_key_block() const { + Block block; + for (const auto& column : _key_columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +Status FlussUnionLakeReader::_accumulate_tail_keys(const Tail& tail, const NextBatch& next_batch, + Block* keys) { + auto builder = MutableBlock::build_mutable_block(_empty_key_block()); + Block batch = _empty_key_block(); + while (true) { + bool eos = false; + batch.clear_column_data(cast_set(_key_columns.size())); + RETURN_IF_ERROR(next_batch(&batch, &eos)); + if (batch.rows() > 0) { + RETURN_IF_ERROR(builder.merge(batch)); + if (cast_set(builder.rows()) > _max_tail_rows) { + // The same yardstick and the same limit the Java side applies while replaying this + // very range: change log records read, not distinct keys. Both halves of one table + // therefore exceed it together, so which of them reports it is not a coin toss. + return Status::InternalError( + "the log tail of bucket {} holds more than {} change log records, the " + "limit " + "set by 'fluss.union_read.max_tail_rows'. Read the table as pure fluss " + "with " + "'fluss.union_read.mode=disabled', wait for tiering to move the tail into " + "the lake, or raise the limit", + tail.bucket_id, _max_tail_rows); + } + } + if (eos) { + *keys = builder.to_block(); + return Status::OK(); + } + } +} + +Status FlussUnionLakeReader::_read_tail_keys(const Tail& tail, Block* keys) { + DORIS_CHECK(keys != nullptr); +#ifdef BE_TEST + if (_test_tail_reader) { + // Stands in for the network read alone. Everything built on it - the accumulation, the + // limit, the cache - is the same code the real read goes through. + Block canned; + RETURN_IF_ERROR(_test_tail_reader(tail, &canned)); + bool delivered = false; + return _accumulate_tail_keys( + tail, + [&](Block* batch, bool* eos) { + if (!delivered) { + *batch = canned; + delivered = true; + } + *eos = true; + return Status::OK(); + }, + keys); + } +#endif + // The tail is read as an ordinary bounded log range, projected down to the key columns. Every + // record counts, including the deletions: a lake row whose key the tail deleted must disappear + // just as surely as one it updated, and what the tail ended up saying is contributed by its own + // range, not by this read. + FlussJniReader reader; + RETURN_IF_ERROR(reader.init({ + .projected_columns = _key_columns, + .conjuncts = {}, + .format = format::FileFormat::JNI, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + })); + // Not the adaptive size the lake half is being read at: this is a bounded pass over key columns + // only, and its batches have nothing to do with how wide the lake's rows turned out to be. + reader.set_batch_size(_runtime_state == nullptr ? TAIL_BATCH_ROWS + : _runtime_state->batch_size()); + + format::SplitReadOptions tail_options; + tail_options.conjuncts = VExprContextSPtrs {}; + tail_options.condition_cache_digest = 0; + tail_options.current_range = tail_scan_range(tail); + tail_options.current_split_format = format::FileFormat::JNI; + + const auto drain = [&]() -> Status { + RETURN_IF_ERROR(reader.prepare_split(tail_options)); + return _accumulate_tail_keys( + tail, [&](Block* batch, bool* eos) { return reader.get_block(batch, eos); }, keys); + }; + const auto status = drain(); + // A fluss connection left open keeps its netty and metadata-updater threads alive for the life + // of the BE process, so the reader is closed on the failing path too. + const auto close_status = reader.close(); + RETURN_IF_ERROR(status); + return close_status; +} + +Status FlussUnionLakeReader::_build_suppression_predicate(const Block& keys) { + std::vector key_positions; + key_positions.reserve(_key_column_indexes.size()); + for (const auto index : _key_column_indexes) { + key_positions.push_back(cast_set(index)); + } + DORIS_CHECK(keys.columns() == key_positions.size()); + auto predicate = std::make_shared(keys, key_positions); + for (size_t i = 0; i < _key_column_indexes.size(); ++i) { + const auto position = key_positions[i]; + // The block this runs against is the table-schema block the lake half returns, so a key + // column is simply the projected column at its own position - already mapped, already of the + // projected type, and therefore comparable with the tail's column of the same name. + predicate->add_child(VSlotRef::create_shared(position, position, -1, _key_columns[i].type, + _key_columns[i].name)); + } + _suppression = VExprContext::create_shared(std::move(predicate)); + RowDescriptor row_desc; + RETURN_IF_ERROR(_suppression->prepare(_runtime_state, row_desc)); + return _suppression->open(_runtime_state); +} + +Status FlussUnionLakeReader::get_block(Block* block, bool* eos) { + DORIS_CHECK(_lake_reader != nullptr); + RETURN_IF_ERROR(_lake_reader->get_block(block, eos)); + return _suppress(block); +} + +Status FlussUnionLakeReader::_suppress(Block* block) { + DORIS_CHECK(block != nullptr); + if (_suppression == nullptr) { + // Every wrapped lake split names the tail that supersedes part of it. Reading one without + // having built that suppression would return the superseded rows a second time. + return Status::InternalError( + "fluss union read: a lake split was read with no log tail bound to it"); + } + const auto rows = block->rows(); + if (rows == 0) { + return Status::OK(); + } + SCOPED_TIMER(_profile.finalize_timer); + int result_column_id = -1; + RETURN_IF_ERROR(_suppression->root()->execute(_suppression.get(), block, &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && result_column_id < cast_set(block->columns())); + const auto& suppressed = + assert_cast(*block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(suppressed.size() == rows); + IColumn::Filter keep(rows, 1); + int64_t dropped = 0; + for (size_t row = 0; row < rows; ++row) { + keep[row] = suppressed[row] == 0; + dropped += suppressed[row] != 0 ? 1 : 0; + } + block->erase(result_column_id); + if (dropped == 0) { + return Status::OK(); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, keep)); + update_counter(_suppressed_rows_counter, dropped); + return Status::OK(); +} + +bool FlussUnionLakeReader::current_split_pruned() const { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->current_split_pruned(); +} + +bool FlussUnionLakeReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->current_split_uses_metadata_count(); +} + +Status FlussUnionLakeReader::abort_split() { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->abort_split(); +} + +Status FlussUnionLakeReader::close() { + _suppression.reset(); + _suppression_tail_spec.clear(); + if (_lake_reader == nullptr) { + return Status::OK(); + } + return _lake_reader->close(); +} + +void FlussUnionLakeReader::set_batch_size(size_t batch_size) { + format::TableReader::set_batch_size(batch_size); + if (_lake_reader != nullptr) { + _lake_reader->set_batch_size(_batch_size); + } +} + +int64_t FlussUnionLakeReader::condition_cache_hit_count() const { + return _lake_reader == nullptr ? 0 : _lake_reader->condition_cache_hit_count(); +} + +} // namespace doris::format::fluss diff --git a/be/src/format_v2/table/fluss_union_lake_reader.h b/be/src/format_v2/table/fluss_union_lake_reader.h new file mode 100644 index 00000000000000..f47f5b856bb5bb --- /dev/null +++ b/be/src/format_v2/table/fluss_union_lake_reader.h @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { + +/** + * A lake split of a tiered primary-key fluss table, minus the rows its log tail has superseded. + * + *

A primary-key table whose rows are being tiered into a lake is read as two halves: the lake, as + * the paimon sibling's own splits, and the change log written after the lake snapshot. The halves + * overlap by key rather than by row - a row the lake still holds may since have been updated or + * deleted in the log - so returning both copies would duplicate rows in a way no row count reveals. + * + *

The split of labour: this reader only SUPPRESSES. It reads the keys of one bucket's log tail, + * once per bucket per BE, and drops the lake rows those keys name. What the tail ended up saying + * about each of those keys is contributed separately, by a `PK_TAIL` fluss range replayed on the + * Java side, so every row is produced exactly once and a key the tail ended by deleting is produced + * not at all. + * + *

The lake half is read entirely by the sibling's reader stack. This class owns a + * PaimonHybridReader and forwards every split to it untouched: native ORC/Parquet, serialized JNI + * splits, deletion vectors, schema evolution and split scheduling are all its answer. The wrapping + * range (`fluss_union`) carries the sibling's own payload beside the tail descriptor for exactly + * that reason. + * + *

Where the filter sits. On the block the child returns, in table-schema terms, not inside + * the child's file scan request. FE keeps the primary-key columns in the scan's tuple whenever it + * plans a union read, so the keys are ordinary projected columns of that block and the column mapper + * has already resolved them - by name or by field id, whichever the lake table needs. Pushing the + * predicate into the file request would buy the non-key columns of the suppressed rows not being + * decoded, which is bounded by the size of the tail, and would cost a second place that resolves key + * columns against a file schema plus a separate implementation for the JNI child. + */ +class FlussUnionLakeReader final : public format::TableReader { +public: + /** One bucket's log tail: the half-open offset range the lake snapshot does not cover yet. */ + struct Tail { + /** Empty on an unpartitioned table, which fluss subscribes to by bucket alone. */ + std::string partition_id; + int32_t bucket_id = 0; + int64_t start_offset = 0; + int64_t stop_offset = 0; + /** Exactly what FE wrote. Also the identity of this tail in the scan node's split cache. */ + std::string spec; + }; + + /** + * The keys of one bucket's log tail, one row per change-log record and one column per key + * column. Held by the scan node's split cache so every split of that bucket shares one read. + */ + struct SuppressionKeys { + Block keys; + }; + + ~FlussUnionLakeReader() override = default; + + Status init(format::TableReadOptions&& options) override; + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; + Status abort_split() override; + Status close() override; + void set_batch_size(size_t batch_size) override; + int64_t condition_cache_hit_count() const override; + + /** + * `partitionId:bucket:start:stop`, as FE encoded it. The partition id is empty rather than a + * sentinel on an unpartitioned table, so that its buckets cannot collide with a partitioned + * table's in the cache. + */ + static Status parse_tail(const std::string& spec, Tail* tail); + + /** The fluss range that reads that tail: one bucket's log over `[start, stop)`, keys only. */ + static TFileRangeDesc tail_scan_range(const Tail& tail); + +private: + /** Fills one batch of tail keys and says whether the tail has been read to its end. */ + using NextBatch = std::function; + + Status _resolve_union_properties(); + Status _prepare_suppression(const format::SplitReadOptions& options); + Status _load_suppression_keys(const format::SplitReadOptions& options, const Tail& tail); + Status _accumulate_tail_keys(const Tail& tail, const NextBatch& next_batch, Block* keys); + Status _read_tail_keys(const Tail& tail, Block* keys); + Status _build_suppression_predicate(const Block& keys); + Status _suppress(Block* block); + Block _empty_key_block() const; + void _init_union_profile(); + + std::unique_ptr _lake_reader; + + /** Positions of the key columns in the projected block, in the order FE listed them. */ + std::vector _key_column_indexes; + /** The projected column descriptions of those same columns, reused rather than rebuilt. */ + std::vector _key_columns; + int64_t _max_tail_rows = 0; + + /** The tail the current predicate was built from; a split of the same bucket reuses it. */ + std::string _suppression_tail_spec; + VExprContextSPtr _suppression; + + RuntimeProfile::Counter* _suppressed_rows_counter = nullptr; + RuntimeProfile::Counter* _tail_keys_read_counter = nullptr; + RuntimeProfile::Counter* _tail_cache_hit_counter = nullptr; + +#ifdef BE_TEST + // Reading the tail needs a fluss cluster. Tests stand in for that one step and drive everything + // built on top of it - the cache, the limit, and the suppression itself. + std::function _test_tail_reader; +#endif +}; + +} // namespace doris::format::fluss diff --git a/be/test/format_v2/table/fluss_union_lake_reader_test.cpp b/be/test/format_v2/table/fluss_union_lake_reader_test.cpp new file mode 100644 index 00000000000000..46356c0090a284 --- /dev/null +++ b/be/test/format_v2/table/fluss_union_lake_reader_test.cpp @@ -0,0 +1,593 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/fluss_union_lake_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format/format_common.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { +namespace { + +using Tail = FlussUnionLakeReader::Tail; + +// The lake half, stood in for. Everything about how a paimon split is read is the sibling's answer; +// what this reader adds is only what happens to the block afterwards, which is what these tests are. +class RecordingLakeReader final : public format::TableReader { +public: + Status prepare_split(const format::SplitReadOptions& options) override { + prepared_ranges.push_back(options.current_range); + _current_split_pruned = prune_next_split; + return Status::OK(); + } + + Status get_block(Block* block, bool* eos) override { + if (blocks.empty()) { + *eos = true; + return Status::OK(); + } + *eos = false; + *block = std::move(blocks.front()); + blocks.erase(blocks.begin()); + return Status::OK(); + } + + Status close() override { + closed = true; + return Status::OK(); + } + + std::vector prepared_ranges; + std::vector blocks; + bool prune_next_split = false; + bool closed = false; +}; + +DataTypePtr int_type() { + return std::make_shared(); +} + +DataTypePtr string_type() { + return std::make_shared(); +} + +format::ColumnDefinition make_column(const std::string& name, DataTypePtr type) { + format::ColumnDefinition column; + column.name = name; + column.type = std::move(type); + return column; +} + +// id (int) / name (string) / amount (int); the primary key is (id, name) unless stated otherwise. +std::vector projected_columns() { + return {make_column("id", int_type()), make_column("name", string_type()), + make_column("amount", int_type())}; +} + +TFileScanRangeParams make_scan_params(std::map fluss_properties) { + TFileScanRangeParams scan_params; + scan_params.__set_fluss_properties(std::move(fluss_properties)); + return scan_params; +} + +TFileScanRangeParams union_scan_params(const std::string& pk_names = "id,name", + const std::string& max_tail_rows = "2000000") { + return make_scan_params({{"fluss.db_name", "db"}, + {"fluss.table_name", "lake_pk"}, + {"fluss.union.pk_names", pk_names}, + {"fluss.union.max_tail_rows", max_tail_rows}}); +} + +TFileRangeDesc wrapped_lake_split(const std::string& tail) { + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss_union"); + table_format_params.__set_fluss_params( + {{"fluss.range_type", "LAKE_SUPPRESS"}, {"fluss.union.tail", tail}}); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_params)); + range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return range; +} + +Status init_reader(FlussUnionLakeReader* reader, TFileScanRangeParams* scan_params, + std::vector columns = projected_columns()) { + return reader->init({ + .projected_columns = std::move(columns), + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + }); +} + +// Replaces the paimon child after init(), which is where the real one is built and asked what it +// makes of the scan's options. +RecordingLakeReader* install_lake_reader(FlussUnionLakeReader* reader) { + auto lake_reader = std::make_unique(); + auto* raw = lake_reader.get(); + reader->_lake_reader = std::move(lake_reader); + return raw; +} + +Block make_block(const std::vector& ids, const std::vector& names, + const std::vector& amounts) { + auto id_column = ColumnInt32::create(); + auto name_column = ColumnString::create(); + auto amount_column = ColumnInt32::create(); + for (size_t row = 0; row < ids.size(); ++row) { + id_column->insert_value(ids[row]); + name_column->insert_data(names[row].data(), names[row].size()); + amount_column->insert_value(amounts[row]); + } + Block block; + block.insert({std::move(id_column), int_type(), "id"}); + block.insert({std::move(name_column), string_type(), "name"}); + block.insert({std::move(amount_column), int_type(), "amount"}); + return block; +} + +// The tail as this reader receives it: one row per change log record, key columns only, in the order +// FE listed the key. +Block make_key_block(const std::vector& ids, const std::vector& names) { + auto id_column = ColumnInt32::create(); + auto name_column = ColumnString::create(); + for (size_t row = 0; row < ids.size(); ++row) { + id_column->insert_value(ids[row]); + name_column->insert_data(names[row].data(), names[row].size()); + } + Block block; + block.insert({std::move(id_column), int_type(), "id"}); + block.insert({std::move(name_column), string_type(), "name"}); + return block; +} + +std::vector ids_of(const Block& block) { + std::vector ids; + const auto& column = assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < column.size(); ++row) { + ids.push_back(column.get_data()[row]); + } + return ids; +} + +// ------------------------------------------------------------------ the tail descriptor + +// The four fields are how a split says which slice of which bucket's log may contradict it. A tail +// read from the wrong bucket suppresses rows nothing has superseded and leaves the superseded ones. +TEST(FlussUnionLakeReaderTest, ParsesTheTailOfBothPartitionedAndUnpartitionedTables) { + Tail partitioned; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail("1234:3:17:42", &partitioned).ok()); + EXPECT_EQ(partitioned.partition_id, "1234"); + EXPECT_EQ(partitioned.bucket_id, 3); + EXPECT_EQ(partitioned.start_offset, 17); + EXPECT_EQ(partitioned.stop_offset, 42); + EXPECT_EQ(partitioned.spec, "1234:3:17:42"); + + Tail unpartitioned; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":3:0:9", &unpartitioned).ok()); + // Empty rather than a sentinel number, so that bucket 3 of an unpartitioned table and bucket 3 + // of some partition cannot become one entry in the cache that holds their keys. + EXPECT_TRUE(unpartitioned.partition_id.empty()); + EXPECT_EQ(unpartitioned.bucket_id, 3); +} + +TEST(FlussUnionLakeReaderTest, RefusesATailItCannotReadInFull) { + Tail tail; + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("3:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("1:3:0:9:extra", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("::0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":bucket:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("x:3:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:0:", &tail).ok()); +} + +// An empty range is never planned: FE leaves a bucket whose lake has caught up with its log +// unwrapped. One arriving here means the two halves were bounded by different offsets, which +// duplicates or loses rows - and suppressing nothing would look exactly like working. +TEST(FlussUnionLakeReaderTest, RefusesATailThatSupersedesNothing) { + Tail tail; + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:9:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:10:9", &tail).ok()); +} + +// The range that reads the tail is synthesized here, not planned by FE, so nothing downstream would +// notice it naming the wrong bucket or the wrong offsets. Pinned as a whole map for that reason. +TEST(FlussUnionLakeReaderTest, ReadsTheTailAsABoundedLogRangeOfThatBucket) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail("77:3:17:42", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + ASSERT_TRUE(range.__isset.table_format_params); + EXPECT_EQ(range.table_format_params.table_format_type, "fluss"); + EXPECT_EQ(range.format_type, TFileFormatType::FORMAT_JNI); + const std::map expected {{"fluss.range_type", "LOG"}, + {"fluss.partition_id", "77"}, + {"fluss.bucket_id", "3"}, + {"fluss.log_start_offset", "17"}, + {"fluss.log_stop_offset", "42"}}; + EXPECT_EQ(range.table_format_params.fluss_params, expected); +} + +// LOG, not PK_TAIL. The suppression set is every key the tail touched, including the ones it ended +// by deleting: a lake row whose key was deleted has to disappear too, and PK_TAIL - which returns +// the surviving state - would not name it. +TEST(FlussUnionLakeReaderTest, ReadsTheTailAsAWholeChangeLogNotAsItsSurvivingState) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":0:0:5", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + EXPECT_EQ(range.table_format_params.fluss_params.at("fluss.range_type"), "LOG"); +} + +// An unpartitioned table's bucket is subscribed to by a different fluss call than a partitioned +// one's, chosen by whether the range carries a partition id at all. +TEST(FlussUnionLakeReaderTest, LeavesOutThePartitionIdOfAnUnpartitionedTable) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":3:0:9", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + EXPECT_EQ(range.table_format_params.fluss_params.count("fluss.partition_id"), 0); +} + +// ------------------------------------------------------------------ the key columns + +// The key columns are the projected ones, reused. FE keeps them in the scan's tuple for this read, +// which is what lets the two halves be compared at all: same tuple, same types, already mapped. +TEST(FlussUnionLakeReaderTest, TakesItsKeyColumnsFromTheProjectionInTheOrderFeListedThem) { + auto scan_params = union_scan_params("name,id"); + FlussUnionLakeReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + // The key order is FE's, not the projection's: the tail's key block arrives in that same order. + ASSERT_EQ(reader._key_column_indexes.size(), 2); + EXPECT_EQ(reader._key_column_indexes[0], 1); + EXPECT_EQ(reader._key_column_indexes[1], 0); + ASSERT_EQ(reader._key_columns.size(), 2); + EXPECT_EQ(reader._key_columns[0].name, "name"); + EXPECT_TRUE(reader._key_columns[0].type->equals(DataTypeString {})); + EXPECT_EQ(reader._key_columns[1].name, "id"); +} + +// The one thing this reader cannot work around. FE promises the key columns are projected whenever +// it plans a union read; if that promise is broken, suppressing nothing returns every superseded +// lake row a second time, and no count or assertion downstream would show it. +TEST(FlussUnionLakeReaderTest, RefusesAScanThatDoesNotProjectAKeyColumn) { + auto scan_params = union_scan_params("id,name"); + FlussUnionLakeReader reader; + const auto status = + init_reader(&reader, &scan_params, + {make_column("id", int_type()), make_column("amount", int_type())}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("name"), std::string::npos); +} + +TEST(FlussUnionLakeReaderTest, RefusesAScanWithoutTheUnionProperties) { + TFileScanRangeParams no_properties; + FlussUnionLakeReader without_properties; + EXPECT_FALSE(init_reader(&without_properties, &no_properties).ok()); + + auto no_names = make_scan_params({{"fluss.db_name", "db"}}); + FlussUnionLakeReader without_names; + EXPECT_FALSE(init_reader(&without_names, &no_names).ok()); + + auto no_limit = make_scan_params({{"fluss.union.pk_names", "id"}}); + FlussUnionLakeReader without_limit; + EXPECT_FALSE(init_reader(&without_limit, &no_limit).ok()); + + auto zero_limit = union_scan_params("id", "0"); + FlussUnionLakeReader with_zero_limit; + EXPECT_FALSE(init_reader(&with_zero_limit, &zero_limit).ok()); + + auto negative_limit = union_scan_params("id", "-1"); + FlussUnionLakeReader with_negative_limit; + EXPECT_FALSE(init_reader(&with_negative_limit, &negative_limit).ok()); +} + +// A COUNT answered from paimon's file metadata would count the rows this reader exists to remove, +// and would do it without ever producing a block to remove them from. +TEST(FlussUnionLakeReaderTest, WithholdsAggregatePushdownFromTheLakeHalf) { + auto scan_params = union_scan_params(); + FlussUnionLakeReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns(), + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader._lake_reader != nullptr); + EXPECT_EQ(reader._lake_reader->_push_down_agg_type, TPushAggOp::type::NONE); + EXPECT_EQ(reader._push_down_agg_type, TPushAggOp::type::COUNT); +} + +// ------------------------------------------------------------------ the suppression + +struct SuppressionFixture { + FlussUnionLakeReader reader; + TFileScanRangeParams scan_params = union_scan_params(); + ShardedKVCache cache {2}; + RecordingLakeReader* lake = nullptr; + std::vector tails_read; + + Status open(Block keys) { + RETURN_IF_ERROR(init_reader(&reader, &scan_params)); + lake = install_lake_reader(&reader); + reader._test_tail_reader = [this, keys](const Tail& tail, Block* out) mutable { + tails_read.push_back(tail.spec); + *out = keys; + return Status::OK(); + }; + return Status::OK(); + } + + format::SplitReadOptions split(const std::string& tail) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = wrapped_lake_split(tail); + return options; + } +}; + +// The whole point, stated once: a lake row whose key the tail touched is gone, and every other lake +// row is untouched. What the tail ended up saying about those keys arrives separately. +TEST(FlussUnionLakeReaderTest, DropsTheLakeRowsTheTailNamesAndKeepsTheRest) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({2, 4}, {"b", "d"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + fixture.lake->blocks.push_back( + make_block({1, 2, 3, 4}, {"a", "b", "c", "d"}, {10, 20, 30, 40})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(ids_of(block), (std::vector {1, 3})); + // The block still has the shape its consumer expects; only rows went away. + EXPECT_EQ(block.columns(), 3); +} + +// The same key column values in different rows are not the same key. Comparing one column of a +// composite key would drop rows nothing superseded. +TEST(FlussUnionLakeReaderTest, MatchesOnTheWholeKeyNotOnOneColumnOfIt) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"b"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + // (1, "a") shares its id with the tail's key and (2, "b") shares its name; neither is that key. + fixture.lake->blocks.push_back(make_block({1, 2, 1}, {"a", "b", "b"}, {10, 20, 30})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_EQ(ids_of(block), (std::vector {1, 2})); +} + +TEST(FlussUnionLakeReaderTest, LeavesASplitWhoseKeysTheTailNeverTouchedAlone) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({7, 8}, {"g", "h"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + fixture.lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_EQ(ids_of(block), (std::vector {1, 2})); +} + +// One bucket's tail is one read, however many of its splits this BE was given. The read is a network +// round trip to fluss; doing it per split would multiply it by the number of files in the bucket. +TEST(FlussUnionLakeReaderTest, ReadsOneBucketsTailOnceHoweverManySplitsItHas) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({2}, {"b"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + EXPECT_EQ(fixture.tails_read, (std::vector {":0:10:20"})); +} + +// Two buckets, two tails, and each split suppressed by its own. Binding them by bucket is what makes +// the tail small; getting the binding wrong is invisible unless the two tails differ. +TEST(FlussUnionLakeReaderTest, SuppressesEachBucketWithItsOwnTail) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + auto* lake = install_lake_reader(&reader); + ShardedKVCache cache {2}; + std::vector tails_read; + reader._test_tail_reader = [&](const Tail& tail, Block* out) { + tails_read.push_back(tail.spec); + *out = tail.bucket_id == 0 ? make_key_block({1}, {"a"}) : make_key_block({2}, {"b"}); + return Status::OK(); + }; + const auto split = [&](const std::string& tail) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = wrapped_lake_split(tail); + return options; + }; + + ASSERT_TRUE(reader.prepare_split(split(":0:10:20")).ok()); + lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + Block bucket0 = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(reader.get_block(&bucket0, &eos).ok()); + EXPECT_EQ(ids_of(bucket0), (std::vector {2})); + + ASSERT_TRUE(reader.prepare_split(split(":1:30:40")).ok()); + lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + Block bucket1 = make_block({}, {}, {}); + ASSERT_TRUE(reader.get_block(&bucket1, &eos).ok()); + EXPECT_EQ(ids_of(bucket1), (std::vector {1})); + EXPECT_EQ(tails_read, (std::vector {":0:10:20", ":1:30:40"})); +} + +// The limit exists because a BE is a long-lived process shared by every query on it. Both halves of +// one table count the same thing - change log records over the same offsets - so a table that is +// over the limit is over it on both sides rather than in whichever half ran first. +TEST(FlussUnionLakeReaderTest, RefusesATailLargerThanTheConfiguredLimit) { + SuppressionFixture over_limit; + over_limit.scan_params = union_scan_params("id,name", "2"); + ASSERT_TRUE(over_limit.open(make_key_block({1, 2, 3}, {"a", "b", "c"})).ok()); + const auto status = over_limit.reader.prepare_split(over_limit.split(":0:10:20")); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("max_tail_rows"), std::string::npos); + + SuppressionFixture at_limit; + at_limit.scan_params = union_scan_params("id,name", "3"); + ASSERT_TRUE(at_limit.open(make_key_block({1, 2, 3}, {"a", "b", "c"})).ok()); + EXPECT_TRUE(at_limit.reader.prepare_split(at_limit.split(":0:10:20")).ok()); +} + +// A tail that reaches this BE in several batches is one tail. Accumulating it batch by batch is +// what keeps the limit from being reached only after the whole of an oversized tail is in memory. +TEST(FlussUnionLakeReaderTest, AccumulatesATailThatArrivesInSeveralBatches) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":0:10:20", &tail).ok()); + + std::vector batches {make_key_block({1}, {"a"}), make_key_block({}, {}), + make_key_block({3}, {"c"})}; + size_t delivered = 0; + Block keys; + ASSERT_TRUE(reader._accumulate_tail_keys( + tail, + [&](Block* batch, bool* eos) { + if (delivered < batches.size()) { + *batch = batches[delivered++]; + } + *eos = delivered >= batches.size(); + return Status::OK(); + }, + &keys) + .ok()); + EXPECT_EQ(keys.rows(), 2); + EXPECT_EQ(ids_of(keys), (std::vector {1, 3})); +} + +// A split that says nothing about which tail supersedes it cannot be read: reading it as-is returns +// the superseded rows a second time. +TEST(FlussUnionLakeReaderTest, RefusesALakeSplitWithNoTailBoundToIt) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + install_lake_reader(&reader); + ShardedKVCache cache {2}; + + const auto prepare = [&](TFileRangeDesc range) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = std::move(range); + return reader.prepare_split(options); + }; + + TFileRangeDesc no_params; + EXPECT_FALSE(prepare(no_params).ok()); + + TTableFormatFileDesc empty_fluss_params; + empty_fluss_params.__set_table_format_type("fluss_union"); + empty_fluss_params.__set_fluss_params({}); + TFileRangeDesc without_tail; + without_tail.__set_table_format_params(empty_fluss_params); + EXPECT_FALSE(prepare(without_tail).ok()); + + TTableFormatFileDesc wrong_type; + wrong_type.__set_table_format_type("fluss_union"); + wrong_type.__set_fluss_params({{"fluss.range_type", "LOG"}, {"fluss.union.tail", ":0:10:20"}}); + TFileRangeDesc not_a_suppression; + not_a_suppression.__set_table_format_params(wrong_type); + EXPECT_FALSE(prepare(not_a_suppression).ok()); + + TTableFormatFileDesc no_tail_key; + no_tail_key.__set_table_format_type("fluss_union"); + no_tail_key.__set_fluss_params({{"fluss.range_type", "LAKE_SUPPRESS"}}); + TFileRangeDesc suppression_without_tail; + suppression_without_tail.__set_table_format_params(no_tail_key); + EXPECT_FALSE(prepare(suppression_without_tail).ok()); +} + +// Reading a block with no suppression built would return the superseded rows. This is the state that +// a future refactor could reach by accident, so it fails rather than passing the block through. +TEST(FlussUnionLakeReaderTest, RefusesToReturnABlockWithNoSuppressionBuilt) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + auto* lake = install_lake_reader(&reader); + lake->blocks.push_back(make_block({1}, {"a"}, {10})); + + Block block = make_block({}, {}, {}); + bool eos = false; + EXPECT_FALSE(reader.get_block(&block, &eos).ok()); +} + +// A pruned split returns no rows, so its bucket's tail is not worth a network round trip. +TEST(FlussUnionLakeReaderTest, DoesNotReadTheTailOfAPrunedSplit) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + fixture.lake->prune_next_split = true; + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + EXPECT_TRUE(fixture.tails_read.empty()); + EXPECT_TRUE(fixture.reader.current_split_pruned()); +} + +// Every split reaches the sibling exactly as it was planned - its path, its byte extent, its paimon +// payload. Rewriting any of it here would reshape a read this reader has no business reshaping. +TEST(FlussUnionLakeReaderTest, ForwardsTheSplitToTheSiblingUntouched) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + auto range = wrapped_lake_split(":0:10:20"); + range.__set_path("/warehouse/db.db/lake_pk/bucket-0/data-1.parquet"); + range.__set_start_offset(4); + range.__set_size(1024); + format::SplitReadOptions options; + options.cache = &fixture.cache; + options.current_range = range; + ASSERT_TRUE(fixture.reader.prepare_split(options).ok()); + + ASSERT_EQ(fixture.lake->prepared_ranges.size(), 1); + EXPECT_EQ(fixture.lake->prepared_ranges[0], range); +} + +TEST(FlussUnionLakeReaderTest, ClosesTheLakeHalf) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + ASSERT_TRUE(fixture.reader.close().ok()); + EXPECT_TRUE(fixture.lake->closed); + EXPECT_TRUE(fixture.reader._suppression == nullptr); +} + +} // namespace +} // namespace doris::format::fluss From bdf964e3344fa0c51cb10dae46fe81e33d2b3480 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 22:02:20 +0800 Subject: [PATCH 32/35] [feat](be) Dispatch a wrapped lake split to the reader that unwraps it A fluss primary-key table read as its lake plus its log tail plans its lake half as the paimon sibling's own split with a suppression descriptor attached, under its own table format name. Without a branch for that name the scanner refuses it outright, and the half of the read that the descriptor exists for never happens. Whether such a split can be read at all is the paimon answer, asked of the paimon payload it carries, rather than a second rule that could start disagreeing with paimon's after the sibling changes how it plans a split. That also keeps the C++ paimon reader's splits on the V1 fallback, where there is no branch for this format -- a clean refusal rather than a lake half read without its suppression. The reader-follows-range logic already covers a scan node holding all three kinds of range at once, which this one does. Checked red by removing the format from the accepted set. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- be/src/exec/scan/file_scanner_v2.cpp | 13 +++++++-- be/test/exec/scan/file_scanner_v2_test.cpp | 31 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 0b0a08c604522e..b38c7cb034674d 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -55,6 +55,7 @@ #include "format_v2/jni/jdbc_reader.h" #include "format_v2/jni/max_compute_jni_reader.h" #include "format_v2/jni/trino_connector_jni_reader.h" +#include "format_v2/table/fluss_union_lake_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" @@ -100,7 +101,11 @@ bool is_supported_table_format(const TFileRangeDesc& range) { return false; } return table_format == "NotSet" || table_format == "tvf" || table_format == "hive" || - table_format == "iceberg" || table_format == "paimon" || table_format == "hudi"; + table_format == "iceberg" || table_format == "paimon" || table_format == "hudi" || + // A lake split of a fluss primary-key table read as its lake plus its log tail. It is the + // paimon sibling's own split, wrapped, so it arrives in whichever form the sibling planned + // it - native Parquet/ORC here, a serialized JNI split below. + table_format == "fluss_union"; } bool is_supported_arrow_table_format(const TFileRangeDesc& range) { @@ -109,7 +114,9 @@ bool is_supported_arrow_table_format(const TFileRangeDesc& range) { bool is_supported_jni_table_format(const TFileRangeDesc& range) { const auto table_format = table_format_name(range); - if (table_format == "paimon") { + // A wrapped lake split is a paimon split in every respect that decides how it is read, so + // whether it can be read here is the paimon answer, asked of the paimon payload it carries. + if (table_format == "paimon" || table_format == "fluss_union") { if (!range.__isset.table_format_params || !range.table_format_params.__isset.paimon_params) { return false; @@ -631,6 +638,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(); } else if (table_format == "fluss") { *reader = std::make_unique(); + } else if (table_format == "fluss_union") { + *reader = std::make_unique(); } else if (table_format == "jdbc") { *reader = std::make_unique(); } else if (table_format == "max_compute") { diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 353c08043adef4..9c924b65f30294 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -318,6 +318,8 @@ TEST(FileScannerV2Test, SupportedFormatMatrix) { {"hive", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"iceberg", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"paimon", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"fluss_union", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"fluss_union", TFileFormatType::FORMAT_ORC, std::nullopt, true}, {"hudi", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"jdbc", TFileFormatType::FORMAT_PARQUET, std::nullopt, false}, {"", TFileFormatType::FORMAT_JNI, std::nullopt, false}, @@ -475,6 +477,35 @@ TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } +// Scenario: a lake split of a fluss primary-key table read as its lake plus its log tail. It is the +// paimon sibling's own split with a suppression descriptor attached, so whether V2 can read it is +// the paimon answer, asked of the paimon payload it carries - not a separate rule that could start +// disagreeing with paimon's after the sibling changes how it plans a split. +TEST(FileScannerV2Test, WrappedLakeSplitsAreSupportedWhereverThePaimonOnesAre) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_JNI); + + auto serialized_split = legacy_paimon_jni_range_without_reader_type(); + serialized_split.table_format_params.__set_table_format_type("fluss_union"); + EXPECT_TRUE(FileScannerV2::is_supported(params, serialized_split)); + + auto native_file_as_jni = range_with_format("fluss_union", TFileFormatType::FORMAT_JNI); + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format("orc"); + native_file_as_jni.table_format_params.__set_paimon_params(paimon_params); + EXPECT_TRUE(FileScannerV2::is_supported(params, native_file_as_jni)); + + // No paimon payload at all is not a split this reader can hand to the sibling. + EXPECT_FALSE(FileScannerV2::is_supported( + params, range_with_format("fluss_union", TFileFormatType::FORMAT_JNI))); + + // The C++ paimon reader stays on the V1 fallback, which has no fluss_union branch: a clean + // refusal rather than a lake half read without its suppression. + auto cpp_split = paimon_cpp_jni_range(); + cpp_split.table_format_params.__set_table_format_type("fluss_union"); + EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_split)); +} + TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) { RuntimeState state {TQueryOptions(), TQueryGlobals()}; RuntimeProfile profile("file_scanner_v2_close_retry"); From 432ff2f33ab034f0d87d2da4e47900aef4c21f20 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Aug 2026 23:07:10 +0800 Subject: [PATCH 33/35] [test](fluss) Read a primary-key lake table on the shapes one bucket hides test_fluss_lake_pk already recorded what such a table reads as, off a read that went to fluss alone -- an answer the merge of lake and log tail has to reproduce row for row. It does: that recorded file is unchanged here, which is the whole point of having recorded it before the merge existed. What a single-bucket fixture cannot show is WHERE each half comes from, since merging per bucket and merging per table are the same arrangement when there is one bucket. Three fixtures separate them: lake_pk_multi spreads nine keys over three buckets and gives only some of them a tail, lake_pk_part stands its three partitions differently towards the lake -- one lake plus tail, one lake alone, one written after tiering stopped and therefore read whole from fluss inside the same scan -- and lake_pk_cold keeps nothing in the log at all. Binding a tail to the wrong bucket suppresses nothing, because a key lives in exactly one bucket, and the rows that tail was meant to replace come back beside their replacements; of the suites here only the multi-bucket one notices. There is deliberately no deletion-vector fixture. Fluss does forward a 'paimon.deletion-vectors.enabled' table property into the paimon table it creates, which was verified against this environment, but its tiering service then writes data files and no index -- and paimon reads such a table as empty. A fixture that reads as empty asserts nothing. Two suites change with the code that landed before this one. `required` no longer refuses a primary-key table, so test_fluss_lake_pk asserts the merge and compares all three modes instead of expecting a refusal, and the refusal block in test_fluss_union_log becomes an assertion that the two table kinds still plan differently. Retrying the fixture build is fixed as well: init.sql drops its fluss database but leaves the paimon tables behind, and fluss refuses to create a lake table over a paimon table that already holds rows, so any attempt that failed halfway used to doom every attempt after it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 10 +- .../fluss/scripts/run-init-sql.sh | 24 ++- .../fluss/sql/init-lake-tail.sql | 24 +++ .../docker-compose/fluss/sql/init.sql | 98 +++++++++ .../fluss/sql/lake-row-counts.sql | 8 +- .../fluss/test_fluss_catalog.out | 6 + .../fluss/test_fluss_lake_only.out | 3 + .../fluss/test_fluss_lake_pk_merge.out | 55 +++++ .../fluss/test_fluss_lake_pk.groovy | 81 ++++--- .../fluss/test_fluss_lake_pk_merge.groovy | 199 ++++++++++++++++++ .../fluss/test_fluss_union_log.groovy | 18 +- 11 files changed, 483 insertions(+), 43 deletions(-) create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index 5a5f9cc4745e49..38851863fcd805 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -103,7 +103,15 @@ The fixtures recreate database `fluss_test` from scratch on every start: | `lake_cold` | lake table read entirely from the lake — no log tail at all | | `lake_types` | lake table with the full type coverage; non-NULL rows tiered, the all-NULL row in the log | | `lake_part` | lake table partitioned by `dt`; only `20260101` has a log tail | -| `lake_pk` | primary-key lake table; its tail updates one tiered row, deletes another and adds a key the lake never saw | +| `lake_pk` | primary-key lake table, one bucket; its tail updates one tiered row, deletes another and adds a key the lake never saw | +| `lake_pk_multi` | primary-key lake table over 3 buckets; the tail reaches some buckets and not others, which is what makes per-bucket binding observable | +| `lake_pk_part` | primary-key lake table partitioned by `dt`: `20260101` is lake + tail, `20260102` is lake only, `20260103` was written after tiering stopped so the lake has never seen it | +| `lake_pk_cold` | primary-key lake table read entirely from the lake — no tail, so nothing to merge | + +There is deliberately no deletion-vector fixture. Fluss does forward a +`paimon.deletion-vectors.enabled` table property into the paimon table it +creates, but its tiering service writes no deletion vector index, and paimon then +reads such a table as empty — see the note in `sql/init.sql`. ### Lake tables are frozen half in, half out diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh index 14156c0ffa7ada..e8b0c8fca59aa3 100755 --- a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -46,7 +46,7 @@ SQL_TIMEOUT_SECONDS=900 ATTEMPTS=3 # Primary-key fixtures whose buckets must have been snapshotted before the # environment counts as ready. See wait_for_kv_snapshots. -SNAPSHOT_TABLES=(pk_basic pk_types pk_part lake_pk) +SNAPSHOT_TABLES=(pk_basic pk_types pk_part lake_pk lake_pk_multi lake_pk_part lake_pk_cold) SNAPSHOT_WAIT_SECONDS=120 # What each lake fixture must hold in paimon before the tail is written -- the @@ -59,6 +59,9 @@ LAKE_EXPECTED_ROWS=( "lake_types=1" "lake_part=3" "lake_pk=3" + "lake_pk_multi=9" + "lake_pk_part=4" + "lake_pk_cold=3" ) LAKE_TIERING_WAIT_SECONDS=300 TIERING_JAR_GLOB='/opt/flink/opt/fluss-flink-tiering-*.jar' @@ -229,12 +232,31 @@ run_sql() { return 0 } +# Removes the paimon side of the lake tables, which is what makes a retry +# possible at all. init.sql drops its fluss database, but that leaves the paimon +# tables where they are -- and fluss refuses to create a lake table whose paimon +# table already exists and holds rows. Without this, an attempt that failed after +# creating one lake table dooms every attempt after it, and the environment comes +# up "failed after 3 attempts" with the real cause three hundred lines up. +# +# Deleting the directory IS dropping the database here: the warehouse is a +# filesystem catalog, mounted writable for exactly this kind of work, and the +# host script empties the same directory before the containers start. +drop_lake_warehouse() { + local warehouse="${FLUSS_PAIMON_WAREHOUSE#file://}" + if [[ -d "${warehouse}/fluss_test.db" ]]; then + echo "Removing the paimon side of the previous attempt: ${warehouse}/fluss_test.db" + rm -rf "${warehouse}/fluss_test.db" + fi +} + run_attempt() { local attempt="$1" # A previous attempt's tiering job would still be consuming the database # init.sql is about to drop. cancel_all_jobs + drop_lake_warehouse start_tiering_job || return 1 run_sql "${MARKER_DIR}/init.sql" "${MARKER_DIR}/init-attempt-${attempt}.log" || return 1 diff --git a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql index 393bb1f325a0fe..0d1dc09efc5469 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql @@ -78,6 +78,30 @@ INSERT INTO lake_pk VALUES (3, 'lp3-hot'), (4, 'lp4-hot'); +-- Three keys out of nine, so the tail reaches some buckets and not others -- +-- which is the whole point of this table. Key 10 is new here, so it also lands +-- in a bucket the lake already holds rows for and must be added to that bucket +-- rather than replacing anything in it. +INSERT INTO lake_pk_multi VALUES + (2, 'm2-hot'), + (10, 'm10-hot'); + +-- 20260101 gets a tail, 20260102 keeps none, and 20260103 is created here -- +-- after tiering stopped -- so the lake never sees it. +INSERT INTO lake_pk_part VALUES + (1, 'pp1a-hot', '20260101'); + +INSERT INTO lake_pk_part VALUES + (5, 'pp3a', '20260103'); + +-- Deletions of rows the lake holds. They are the case a suppression set has to +-- cover but a merge of surviving rows cannot: the key is gone from the tail's +-- own result, and the lake row it removes would otherwise stay. SET 'execution.runtime-mode' = 'batch'; DELETE FROM lake_pk WHERE id = 1; +DELETE FROM lake_pk_multi WHERE id = 7; +DELETE FROM lake_pk_part WHERE id = 2 AND dt = '20260101'; SET 'execution.runtime-mode' = 'streaming'; + +-- lake_pk_cold gets nothing: it is the fixture for a primary-key table the lake +-- already holds in full, where planning must wrap no split and read no tail. diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index c32128b0e5a07c..4245c7a7797277 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -468,3 +468,101 @@ INSERT INTO lake_pk VALUES INSERT INTO lake_pk VALUES (2, 'lp2-lake'); + +-- --------------------------------------------------------------------------- +-- lake_pk_multi: the same table over three buckets. A primary-key table is +-- merged with its log tail per BUCKET, and a single-bucket fixture cannot tell +-- that apart from merging per table: with one bucket the two are the same +-- arrangement. Here they are not -- the tail below touches some buckets and not +-- others, so a lake split may only be filtered by the tail of ITS OWN bucket. +-- Binding one bucket's tail to another's split suppresses nothing (a key lives +-- in exactly one bucket), and the rows that tail was meant to replace come back +-- as duplicates. +-- +-- Nine keys, because which bucket a key lands in is fluss's hash of it and not +-- ours to choose: enough of them to be spread over all three, few enough to +-- read. Row 5 is updated before tiering, so the lake half already holds a +-- merged view here too. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_multi ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_multi VALUES + (1, 'm1'), + (2, 'm2'), + (3, 'm3'), + (4, 'm4'), + (5, 'm5'), + (6, 'm6'), + (7, 'm7'), + (8, 'm8'), + (9, 'm9'); + +INSERT INTO lake_pk_multi VALUES + (5, 'm5-lake'); + +-- --------------------------------------------------------------------------- +-- lake_pk_part: partitioned primary-key table tiered into paimon. The tail is +-- written into one partition only and a third partition is created after +-- tiering has stopped, so one query reads all three ways a partition can stand +-- with respect to the lake: 20260101 is lake plus tail, 20260102 is lake alone, +-- and 20260103 exists in fluss only -- the lake has never heard of it, so its +-- buckets have to be read whole from fluss inside the very same scan. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_part ( + id INT NOT NULL, + name STRING, + dt STRING NOT NULL, + PRIMARY KEY (id, dt) NOT ENFORCED +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_part VALUES + (1, 'pp1a', '20260101'), + (2, 'pp1b', '20260101'), + (3, 'pp2a', '20260102'), + (4, 'pp2b', '20260102'); + +-- --------------------------------------------------------------------------- +-- lake_pk_cold: a primary-key lake table with nothing left in the log. It gets +-- no tail at all, so planning must wrap no lake split and emit no tail range: +-- the merge has to cost nothing when there is nothing to merge, and a reader +-- that always builds a suppression set would still return the right rows here +-- while doing the work -- only the plan shows the difference. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_cold ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_cold VALUES + (1, 'c1'), + (2, 'c2'), + (3, 'c3'); + +-- No deletion-vector fixture, and the reason is worth recording where the next +-- person to want one will look. Fluss does forward a 'paimon.*' table property +-- into the paimon table it creates, so a table declared with +-- 'paimon.deletion-vectors.enabled' = 'true' really is created with deletion +-- vectors on -- that much was verified against this environment. What the +-- tiering service then writes into it is data files and no index: no deletion +-- vector is ever produced, and paimon's own reader answers COUNT(*) with 0 on a +-- table whose files are sitting right there. A fixture that reads as empty +-- proves nothing, so deletion vectors under the merge stay covered by the BE +-- unit tests until the tiering side writes them. diff --git a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql index f42f0fc198d5fe..d1ab414d1623a2 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql @@ -49,4 +49,10 @@ SELECT CONCAT('LAKEROWS:lake_types=', CAST(COUNT(*) AS STRING)) FROM lake_types UNION ALL SELECT CONCAT('LAKEROWS:lake_part=', CAST(COUNT(*) AS STRING)) FROM lake_part UNION ALL -SELECT CONCAT('LAKEROWS:lake_pk=', CAST(COUNT(*) AS STRING)) FROM lake_pk; +SELECT CONCAT('LAKEROWS:lake_pk=', CAST(COUNT(*) AS STRING)) FROM lake_pk +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_multi=', CAST(COUNT(*) AS STRING)) FROM lake_pk_multi +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_part=', CAST(COUNT(*) AS STRING)) FROM lake_pk_part +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_cold=', CAST(COUNT(*) AS STRING)) FROM lake_pk_cold; diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out index a3dfae055cc802..fe067cfb830b31 100644 --- a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out +++ b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out @@ -10,6 +10,9 @@ lake_cold lake_log lake_part lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part lake_types log_basic log_empty @@ -60,6 +63,9 @@ lake_cold lake_log lake_part lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part lake_types log_basic log_empty diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out index 0a56be161de721..2dd86d2b8b6136 100644 --- a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out @@ -106,6 +106,9 @@ lake_cold lake_log lake_part lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part lake_types log_basic log_empty diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out new file mode 100644 index 00000000000000..e60da9c7ef2805 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !multi_lake -- +1 m1 +2 m2 +3 m3 +4 m4 +5 m5-lake +6 m6 +7 m7 +8 m8 +9 m9 + +-- !multi_rows -- +1 m1 +10 m10-hot +2 m2-hot +3 m3 +4 m4 +5 m5-lake +6 m6 +8 m8 +9 m9 + +-- !multi_count -- +9 + +-- !multi_updated -- +m2-hot + +-- !multi_deleted -- + +-- !multi_new -- +m10-hot + +-- !part_rows -- +1 pp1a-hot 20260101 +3 pp2a 20260102 +4 pp2b 20260102 +5 pp3a 20260103 + +-- !part_lake_only -- +3 pp2a +4 pp2b + +-- !part_fluss_only -- +5 pp3a + +-- !part_with_tail -- +1 pp1a-hot + +-- !cold_rows -- +1 c1 +2 c2 +3 c3 + diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy index 1505b3ed76ce0f..678167b8efc802 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy @@ -19,18 +19,20 @@ // // Its two halves cannot be concatenated the way a log table's are: the log // carries updates and deletes of rows the lake already holds, so reading both -// and adding them up returns superseded and deleted rows. Merging them BY KEY is -// not implemented, so such a table is read from fluss alone -- and that read is -// the WHOLE table, not a part of it, because fluss keeps a primary-key table's -// state in its own kv store and tiering copies rows into the lake rather than -// moving them out. (A log table has no such guarantee, which is why IT is read -// as a union and refused when the lake cannot be reached.) +// and adding them up returns superseded and deleted rows. They are merged BY +// KEY instead -- the lake half is read column by column and a lake row is +// dropped when the log tail that follows it touched its key at all, while the +// tail contributes the state it ended in. // -// This suite is therefore two things at once. It checks today's read, and it -// pins the answer a future lake+log merge will have to reproduce row for row: -// the same queries, over a fixture whose lake and log deliberately disagree in -// every way they can. When that merge lands, these recorded results must not -// change -- that is what recording them is for. +// The results below were recorded before that merge existed, off a read that +// went to fluss alone. That read is the WHOLE table rather than a part of it, +// because fluss keeps a primary-key table's state in its own kv store and +// tiering copies rows into the lake rather than moving them out -- so it is a +// second, independent answer to every query here, and the merge has to +// reproduce it row for row. Nothing in the RESULTS says which one ran, which is +// exactly what makes them a baseline; the plan says it, and this suite asserts +// both. When these recorded blocks change, the merge has broken -- that is what +// recording them is for. // // Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and // init-lake-tail.sql, and are frozen -- the tiering service is stopped before @@ -81,6 +83,7 @@ suite("test_fluss_lake_pk", "p0,external") { // that picks between them is randomised by the fuzzy mode this pipeline runs. sql """set enable_file_scanner_v2 = true""" + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } def planOf = { String query -> return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") } @@ -96,34 +99,48 @@ suite("test_fluss_lake_pk", "p0,external") { // would return the deleted row 1, two versions of row 3, and row 4. order_qt_front_door """select id, name from lake_pk""" - // --- auto falls back to exactly what disabled asks for ------------------- - // The same rows out of the same code path, not "something equivalent": this - // result and the one above have to stay identical to each other, and a - // fallback that quietly read something else would break one of them. + // --- the merge lands on what a fluss-only read returns -------------------- + // The same rows out of an entirely different reader: this block and the one + // above have to stay identical to each other. That is the whole argument for + // the merge being right, because fluss serves a primary-key table in full and + // has no lake half to get wrong. order_qt_fluss_only """select id, name from ${flussOnlyCatalog}.fluss_test.lake_pk""" - // --- the plan says the lake was not read --------------------------------- - // Nothing in the RESULT distinguishes this read from a correct merge -- that - // is exactly what makes it a baseline -- so only the plan can say which one - // ran. One bucket, one primary-key range, no lake splits. + // --- the plan says the lake really was read ------------------------------ + // Without this the suite would still pass if the merge quietly stopped + // happening and the read fell back to fluss alone -- the rows would be the + // same ones. One bucket: one lake split, suppressed by the tail of that + // bucket, and one range replaying the tail itself. def plan = planOf("""select * from lake_pk""") assertTrue( - plan.contains("flussScan: unionRead=no, lakeSplits=0, logRanges=0, pkRanges=1, mode=auto"), - "not planned as a fluss-only primary-key read of one bucket: ${plan}") + plan.contains("flussScan: unionRead=yes, lakeSplits=1, suppressedLakeSplits=1, " + + "logRanges=0, pkRanges=0, pkTailRanges=1, mode=auto"), + "not planned as a lake+tail merge of one bucket: ${plan}") - // --- required refuses rather than falling back --------------------------- - // That mode exists so a union-read test cannot pass without a union read. No - // primary-key table can satisfy it yet, and the refusal has to say so -- - // "wait for tiering to commit" would send the reader down a dead end. - test { - sql """select * from ${requiredCatalog}.fluss_test.lake_pk""" - exception "not implemented yet" - } + // --- required merges rather than refusing -------------------------------- + // That mode exists so a union-read test cannot pass by falling back. It used + // to refuse a primary-key table outright; now it has to produce the merge, + // and produce the same rows as the two modes that may fall back. + def requiredPlan = planOf("""select * from ${requiredCatalog}.fluss_test.lake_pk""") + assertTrue(requiredPlan.contains("unionRead=yes"), "required did not merge: ${requiredPlan}") + assertTrue(requiredPlan.contains("mode=required"), "unexpected mode: ${requiredPlan}") + + // Three modes, three catalogs, one answer -- compared here rather than by + // three recorded blocks, because what is being asserted is that they AGREE. + // Two identical recordings only look alike to a reader. + def query = { String catalog -> "select id, name from ${catalog}.fluss_test.lake_pk order by id" } + def merged = rowsOf(query(autoCatalog)) + assertEquals(rowsOf(query(flussOnlyCatalog)), merged, + "auto and disabled disagree on lake_pk") + assertEquals(rowsOf(query(requiredCatalog)), merged, + "required and auto disagree on lake_pk") // --- the ordinary things still work on this path ------------------------- - // Projection, predicates and count all go through the kv snapshot merged with - // the change log, which is a different reader from the log tables' and is - // exercised here for a lake table. + // Projection, predicates and count all run over the merged pair. The + // projection one carries a second load: the merge needs the key column to + // suppress by, and the engine keeps it in the scan for that reason alone -- + // so a single-column result here is also the assertion that the kept column + // stays out of the answer. order_qt_count """select count(*) from lake_pk""" order_qt_projection """select name from lake_pk where id = 3""" // The deleted row stays gone under a predicate too, not only in a full scan: diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy new file mode 100644 index 00000000000000..437c913c25c662 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Merging a primary-key table with its log tail, in the shapes where the +// merge can be got wrong in ways one bucket cannot show. +// +// test_fluss_lake_pk covers what the merge IS, over a single-bucket table. The +// fixtures here cover WHERE each half comes from: +// lake_pk_multi three buckets, tail in some of them -- a lake split may only +// be filtered by the tail of its own bucket; +// lake_pk_part three partitions, one lake+tail, one lake only, one that the +// lake has never seen and that is therefore read whole from +// fluss inside the same scan; +// lake_pk_cold nothing left in the log, so nothing to merge and nothing to +// plan for. +// +// Every table is read twice: through the merge, and through the fluss-only read +// that serves a primary-key table in full. The two have no code in common past +// planning, so their agreeing is the argument -- the recorded blocks alone +// cannot say whether both are wrong the same way. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// init-lake-tail.sql, and are frozen: tiering is stopped before the tail is +// written, so the halves stay apart for as long as the environment lives. +suite("test_fluss_lake_pk_merge", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String mergeCatalog = "test_fluss_lake_pk_merge" + String flussOnlyCatalog = "test_fluss_lake_pk_merge_off" + + // required, not auto: a fallback here would read the right rows out of fluss + // alone and every recorded block would still match, so the mode that cannot + // fall back is the one that makes this suite about the merge. + sql """drop catalog if exists ${mergeCatalog}""" + sql """ + create catalog ${mergeCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${mergeCatalog}""" + sql """use fluss_test""" + // The C++ glue exists only for the v2 file scanner, and the session variable + // that picks between them is randomised by the fuzzy mode this pipeline runs. + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + // The load-bearing check: the merge and a reader that never touches the lake + // agree row for row. It stays in the code rather than becoming a second + // recorded block, because what is asserted is the agreement -- two identical + // recordings only look alike to whoever reads them. + def compareModes = { String query -> + def merged = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, merged, + "merge and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nmerged=${merged}") + } + + // --- three buckets, and a tail that reaches only some of them ------------- + // Which bucket a key lands in is fluss's hash of it, so the numbers below are + // bounds rather than values. What they have to say is that the tail was split + // across buckets and applied to those buckets only: some lake splits carry a + // suppression set and some do not. Binding a tail to the wrong bucket + // suppresses nothing at all -- a key lives in exactly one bucket -- and the + // rows it was meant to replace come back beside their replacements. + def multiPlan = planOf("""select * from lake_pk_multi""") + assertTrue(multiPlan.contains("unionRead=yes"), "not a merge: ${multiPlan}") + def multiLakeSplits = countIn(multiPlan, "lakeSplits") + def multiSuppressed = countIn(multiPlan, "suppressedLakeSplits") + def multiTailRanges = countIn(multiPlan, "pkTailRanges") + assertTrue(multiLakeSplits >= 2, + "fixture no longer spreads over buckets, so nothing here tests binding: ${multiPlan}") + assertTrue(multiTailRanges >= 2, + "fixture tail no longer spans buckets, so nothing here tests binding: ${multiPlan}") + assertTrue(multiSuppressed >= 1 && multiSuppressed < multiLakeSplits, + "suppression is not per bucket: ${multiSuppressed} of ${multiLakeSplits} splits: ${multiPlan}") + // Every bucket of this table is in the lake, so none is read whole from fluss. + assertEquals(0, countIn(multiPlan, "pkRanges"), "a bucket was read whole: ${multiPlan}") + + // The lake half on its own: nine keys as tiered, row 5 already updated. + order_qt_multi_lake """select id, name from lake_pk_multi\$lake""" + // The table: row 2 updated by the tail, row 7 deleted by it, row 10 added -- + // each in whatever bucket its key hashes to. + order_qt_multi_rows """select id, name from lake_pk_multi""" + compareModes("select id, name from lake_pk_multi order by id") + order_qt_multi_count """select count(*) from lake_pk_multi""" + // A predicate on a key the tail replaced, and one on a key it deleted: the + // filtered path has to suppress exactly as the full scan does. + order_qt_multi_updated """select name from lake_pk_multi where id = 2""" + order_qt_multi_deleted """select name from lake_pk_multi where id = 7""" + order_qt_multi_new """select name from lake_pk_multi where id = 10""" + + // --- partitions, each standing differently towards the lake --------------- + // 20260101 is lake plus tail, 20260102 is lake alone, 20260103 was created + // after tiering stopped and the lake has never heard of it. All three are read + // by one scan, which therefore mixes suppressed lake splits, plain lake + // splits, and buckets read whole out of fluss. + def partPlan = planOf("""select * from lake_pk_part""") + assertTrue(partPlan.contains("unionRead=yes"), "not a merge: ${partPlan}") + assertTrue(countIn(partPlan, "lakeSplits") >= 2, "lake half lost a partition: ${partPlan}") + assertTrue(countIn(partPlan, "suppressedLakeSplits") >= 1, + "the partition with a tail was not suppressed: ${partPlan}") + assertTrue(countIn(partPlan, "pkTailRanges") >= 1, "no tail replayed: ${partPlan}") + assertTrue(countIn(partPlan, "pkRanges") >= 1, + "the partition the lake never saw was not read from fluss: ${partPlan}") + + order_qt_part_rows """select id, name, dt from lake_pk_part""" + compareModes("select id, name, dt from lake_pk_part order by dt, id") + + // Pruned to the partition the lake holds in full: lake splits, no tail, and + // nothing suppressed. Pruning is per partition on the fluss half and by pushed + // predicate on the paimon half, so a plan that keeps the tail here has bound + // the two halves at table level. + def tieredPlan = planOf("""select * from lake_pk_part where dt = '20260102'""") + assertTrue(countIn(tieredPlan, "lakeSplits") >= 1, "lake half pruned away: ${tieredPlan}") + assertEquals(0, countIn(tieredPlan, "suppressedLakeSplits"), + "a partition with no tail was suppressed: ${tieredPlan}") + assertEquals(0, countIn(tieredPlan, "pkTailRanges"), "tail read for nothing: ${tieredPlan}") + order_qt_part_lake_only """select id, name from lake_pk_part where dt = '20260102'""" + + // Pruned to the partition the lake has never seen: no lake half at all, and + // its buckets read whole from fluss. + def flussOnlyPartPlan = planOf("""select * from lake_pk_part where dt = '20260103'""") + assertEquals(0, countIn(flussOnlyPartPlan, "lakeSplits"), + "a partition the lake never saw got lake splits: ${flussOnlyPartPlan}") + assertTrue(countIn(flussOnlyPartPlan, "pkRanges") >= 1, + "the partition was not read from fluss: ${flussOnlyPartPlan}") + order_qt_part_fluss_only """select id, name from lake_pk_part where dt = '20260103'""" + + // Pruned to the partition that has a tail. + def tailPartPlan = planOf("""select * from lake_pk_part where dt = '20260101'""") + assertTrue(countIn(tailPartPlan, "suppressedLakeSplits") >= 1, + "the tail of this partition was lost: ${tailPartPlan}") + assertEquals(1, countIn(tailPartPlan, "pkTailRanges"), "unexpected tail count: ${tailPartPlan}") + order_qt_part_with_tail """select id, name from lake_pk_part where dt = '20260101'""" + + // --- nothing left in the log --------------------------------------------- + // The lake holds this table in full. Merging costs nothing when there is + // nothing to merge, and only the plan can say so: a reader that built a + // suppression set out of an empty tail would return these same rows while + // opening a log scanner per bucket to do it. + def coldPlan = planOf("""select * from lake_pk_cold""") + assertTrue(coldPlan.contains("unionRead=yes"), "not a merge: ${coldPlan}") + assertTrue(countIn(coldPlan, "lakeSplits") >= 1, "no lake splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "suppressedLakeSplits"), + "a table with no tail still wraps splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "pkTailRanges"), "a tail was read for nothing: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "pkRanges"), "a bucket was read whole: ${coldPlan}") + order_qt_cold_rows """select id, name from lake_pk_cold""" + compareModes("select id, name from lake_pk_cold order by id") + + // Deletion vectors are not covered here and the reason is in init.sql: fluss + // does create the paimon table with them on, but its tiering service writes + // no deletion vector index, and paimon then reads such a table as empty. A + // fixture that reads as empty asserts nothing. Deletion vectors under the + // suppression filter stay covered by the BE unit tests. + + sql """switch internal""" + sql """drop catalog if exists ${mergeCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy index d612c49cb04c9f..b5acc6a2802343 100644 --- a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy @@ -162,14 +162,16 @@ suite("test_fluss_union_log", "p0,external") { order_qt_part_with_tail """select id from lake_part where dt = '20260101'""" // --- a primary-key table is not a log table ------------------------------ - // Its halves have to be merged BY KEY, which is not implemented, so `required` - // refuses it rather than falling back. What such a table DOES read as lives in - // test_fluss_lake_pk, together with the baseline that merge will have to - // reproduce. - test { - sql """select * from lake_pk""" - exception "not implemented yet" - } + // Concatenating its halves the way this suite's tables are concatenated would + // return superseded and deleted rows, so it is merged BY KEY instead -- a + // different plan out of the same mode, asserted here only so that the two + // shapes cannot quietly become one. What such a table reads as lives in + // test_fluss_lake_pk and test_fluss_lake_pk_merge. + def pkPlan = planOf("""select * from lake_pk""") + assertTrue(pkPlan.contains("unionRead=yes"), "not a union read: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "logRanges"), + "a primary-key table planned a plain log range: ${pkPlan}") + assertTrue(countIn(pkPlan, "pkTailRanges") >= 1, "no tail replayed: ${pkPlan}") // --- required does not mean "every table has a lake" --------------------- // A table with no lake at all is not an error in required mode: there is From c052dfc0d22d21a9d2c91bffe4425138587fd5c6 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 4 Aug 2026 00:45:26 +0800 Subject: [PATCH 34/35] [feat](fluss) Refuse a partition column fluss cannot keep the value of A fluss partition carries its value nowhere but in its own name, and fluss allows only ASCII letters, digits, '_' and '-' there -- so a value holding anything else is rewritten on the way in. A FLOAT 1.5 is named 1_5; a TIMESTAMP 2026-01-01 01:02:03 is named 2026-01-01-01-02-03. The substitution is many-to-one and Doris is handed the name, not the value, so nothing reads those back. Fluss creates such a table without complaint and DESC looks ordinary, so until now the first sign of trouble came from fe-core's partition parser: "failed to convert partition [1_5] to list partition" -- naming neither the column, nor its type, nor fluss, nor what to do instead. Decide it from the table's schema instead, before the partitions are listed, so the answer is the same whether the table has any yet or not, and say which column and which type. CHAR, STRING, BOOLEAN, the integer family and DATE are kept verbatim and pass; BINARY and BYTES are named with the hex text of their bytes, which reads back as the text it is unless enable.mapping.varbinary asks for a VARBINARY column, so their verdict is the one the catalog decides. Every verdict was established against a fluss cluster rather than reasoned from the naming rules, and the switch is exhaustive over fluss's type roots: a type a future release adds to its partition-key whitelist is refused with a message rather than waved through into the parser. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../fluss/FlussConnectorMetadata.java | 39 ++++ .../fluss/FlussPartitionColumnTypes.java | 95 +++++++++ .../fluss/FlussConnectorMetadataTest.java | 130 +++++++++++++ .../fluss/FlussPartitionColumnTypesTest.java | 184 ++++++++++++++++++ 4 files changed, 448 insertions(+) create mode 100644 fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java create mode 100644 fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java index 6f3274679be2d1..b4c25e6f40632c 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -39,6 +39,7 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypeRoot; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -335,6 +336,10 @@ public List listPartitions(ConnectorSession session, // an error there, and "this table has no partitions" is already known from the handle. return Collections.emptyList(); } + // Before the remote call and before any name is rendered, because the answer depends on the + // table's schema alone: a partition column whose value fluss cannot store verbatim in a partition + // name makes every partition of this table unreadable, whether it has any yet or not. + rejectUnreadablePartitionColumns(flussHandle, partitionKeys); List flussPartitions = adminOps.listPartitionInfos(flussHandle.toTablePath()); List result = new ArrayList<>(flussPartitions.size()); @@ -349,6 +354,40 @@ public List listPartitions(ConnectorSession session, return result; } + /** + * Refuses a table whose partition values cannot be read back out of the names fluss stores them in. + * + *

This is the only place it can be said well. Further down, the name is all there is: fe-core's + * parser sees {@code 1_5}, has a FLOAT column to put it in, and reports that it failed to convert a + * partition — naming neither the column nor fluss nor the property that would help. Here the column + * and its fluss type are both still in hand. + * + *

Refusing is the answer rather than guessing because the rewriting is many-to-one: {@code 1_5} + * was {@code 1.5}, and {@code 01-02-03} was {@code 01:02:03}, and neither the connector nor fluss + * itself can say which character came back out. + */ + private void rejectUnreadablePartitionColumns(FlussTableHandle handle, List partitionKeys) { + Map keyColumnTypes = handle.getKeyColumnTypes(); + for (String partitionKey : partitionKeys) { + DataType type = keyColumnTypes.get(partitionKey); + if (type == null) { + // The handle records a type for every partition column of the table it was built from, so + // a missing one means this handle and that schema have come apart. Guessing "readable" + // here would put the mangled name back on the path this method exists to close. + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' lists '" + partitionKey + "' as a partition column" + + " but carries no type for it; refresh the catalog and try again."); + } + String rejection = FlussPartitionColumnTypes.rejection(type, typeMappingOptions); + if (rejection != null) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read: its partition column '" + partitionKey + + "' has fluss type " + type + ", and " + rejection + ". Partition columns of type " + + FlussPartitionColumnTypes.READABLE_TYPES + " are stored as written."); + } + } + } + /** * The table's row count, when fluss has one. * diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java new file mode 100644 index 00000000000000..253bfb9720fa79 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; + +/** + * Which column types a fluss table can be PARTITIONED by and still be readable here. + * + *

A fluss partition carries its value nowhere but in its own name, and fluss restricts what that name + * may contain to ASCII letters, digits, {@code _} and {@code -} — so a value holding anything else is + * rewritten on the way in. A FLOAT {@code 1.5} is named {@code 1_5}; a TIMESTAMP + * {@code 2026-01-01 01:02:03.0} is named {@code 2026-01-01-01-02-03_0}. Nothing reads those back: the + * substitution is many-to-one (a {@code _} was a {@code .} or a {@code :}), and Doris is handed the name, + * not the value. + * + *

Which is why this is a refusal and not a conversion. Left alone, such a table reaches fe-core's + * partition parser and dies there with {@code failed to convert partition [1_5] to list partition} — a + * message that names neither the column nor fluss nor what to do about it, on a table whose {@code DESC} + * looks perfectly ordinary. The verdict below is asked before the partitions are listed, so the answer + * depends on the table's schema alone and is the same whether the table has partitions yet or not. + * + *

Not to be confused with {@link FlussUnionKeyTypes#partitionColumnRejection}, which asks a stricter + * and later question: whether a LAKE split can be matched to a fluss partition by comparing two engines' + * renderings of the same value. That one allows STRING only. This one asks whether Doris can read the + * partition at all, and its answer is a precondition for the other's. + */ +final class FlussPartitionColumnTypes { + + /** The types whose value survives fluss's partition naming, for the error message. */ + static final String READABLE_TYPES = "CHAR, STRING, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT and DATE"; + + private FlussPartitionColumnTypes() { + } + + /** + * Why {@code type} cannot be a partition column of a readable table, or null when it can. + * + *

The switch is exhaustive over fluss's type roots on purpose: a type a future fluss release adds + * to its own partition-key whitelist lands in the default branch and is refused with a message, rather + * than being waved through into fe-core's parser by a rule written before it existed. + * {@link FlussPartitionColumnTypesTest} fails the build when that happens. + * + *

BINARY and BYTES are the one verdict that depends on the catalog: fluss names their partitions + * with the hex text of the bytes, which is a perfectly good STRING and not a VARBINARY at all. + */ + static String rejection(DataType type, FlussTypeMapping.Options options) { + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case DATE: + return null; + case BINARY: + case BYTES: + return options.isMapBinaryToVarbinary() + ? "fluss names such a partition with the hex text of the bytes, which this catalog" + + " cannot read back as the VARBINARY column that '" + + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY + "=true' asks for;" + + " turning that property off reads the column, and the partition, as text" + : null; + case FLOAT: + case DOUBLE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "fluss keeps a partition's value only in its name, where it rewrites every character" + + " a name may not contain — 1.5 is named 1_5 and 2026-01-01 01:02:03 is named" + + " 2026-01-01-01-02-03 — so the value that was written cannot be read back"; + default: + // ARRAY, MAP and ROW among them: fluss refuses these as partition keys itself, and this + // branch is what keeps that from being the only thing standing in the way. + return "the fluss connector does not know how a partition of this type is named"; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java index 76c949b81bf36f..3a836c41e877aa 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java @@ -22,6 +22,7 @@ import org.apache.doris.connector.api.ConnectorTableSchema; import org.apache.doris.connector.api.ConnectorTableStatistics; import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; import org.apache.doris.connector.api.handle.ConnectorColumnHandle; import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.thrift.TTableDescriptor; @@ -58,6 +59,7 @@ public class FlussConnectorMetadataTest { private static final TablePath LOG_TABLE = TablePath.of("db", "log_table"); private static final TablePath PK_TABLE = TablePath.of("db", "pk_table"); + private static final TablePath PART_TABLE = TablePath.of("db", "part_table"); private static FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { return metadata(adminOps, FlussTypeMapping.Options.DEFAULT); @@ -328,6 +330,134 @@ public void anUnpartitionedTableNeverAsksTheClusterForPartitions() { "no partition call should have been made, calls were: " + adminOps.calls); } + /** + * A partition column of every type fluss allows AND Doris can read back, in one table. The values + * below are the ones a fluss cluster really names these partitions with — verified against one — + * because the whole question here is whether the name survives the round trip. + */ + private static RecordingFlussAdminOps withEveryReadablePartitionType() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PART_TABLE, FlussTestTables.builder(PART_TABLE) + .column("id", DataTypes.INT()) + .column("p_str", DataTypes.STRING()) + .column("p_char", DataTypes.CHAR(2)) + .column("p_bool", DataTypes.BOOLEAN()) + .column("p_tiny", DataTypes.TINYINT()) + .column("p_small", DataTypes.SMALLINT()) + .column("p_int", DataTypes.INT()) + .column("p_big", DataTypes.BIGINT()) + .column("p_date", DataTypes.DATE()) + .column("p_bin", DataTypes.BINARY(2)) + .partitionedBy("p_str", "p_char", "p_bool", "p_tiny", "p_small", "p_int", "p_big", + "p_date", "p_bin") + .buckets(1) + .build()); + adminOps.partitionsByTable.put(PART_TABLE, Collections.singletonList( + partition(1L, "p_str", "cn", "p_char", "c1", "p_bool", "true", "p_tiny", "1", + "p_small", "10", "p_int", "100", "p_big", "1000", "p_date", "2026-01-01", + "p_bin", "0102"))); + return adminOps; + } + + /** A table partitioned by one column of the given type, and nothing else of interest. */ + private static RecordingFlussAdminOps withPartitionColumnOfType( + org.apache.fluss.types.DataType type) { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PART_TABLE, FlussTestTables.builder(PART_TABLE) + .column("id", DataTypes.INT()) + .column("p", type) + .partitionedBy("p") + .buckets(1) + .build()); + adminOps.partitionsByTable.put(PART_TABLE, + Collections.singletonList(partition(1L, "p", "2026-01-01-01-02-03"))); + return adminOps; + } + + @Test + public void everyPartitionTypeWhoseNameSurvivesIsListed() { + RecordingFlussAdminOps adminOps = withEveryReadablePartitionType(); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(1, partitions.size()); + Assertions.assertEquals("p_str=cn/p_char=c1/p_bool=true/p_tiny=1/p_small=10/p_int=100/" + + "p_big=1000/p_date=2026-01-01/p_bin=0102", partitions.get(0).getPartitionName()); + } + + /** + * Fluss stores a partition's value only in its name, and rewrites the characters a name may not + * hold: a TIMESTAMP {@code 2026-01-01 01:02:03} is named {@code 2026-01-01-01-02-03}. Handing that + * on gets it as far as fe-core's partition parser, which fails with the name and nothing else — + * no column, no type, no fluss. So the refusal belongs here, where all three are still known. + */ + @Test + public void partitionValueFlussRewroteIsRefusedByName() { + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.TIMESTAMP(3)); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue(failure.getMessage().contains("db.part_table"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("'p'"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("TIMESTAMP(3)"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("cannot be read back"), failure.getMessage()); + // Refused from the schema alone, so a table with no partitions yet is refused the same way and + // the cluster is not asked a question whose answer could not be used. + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "the partitions should not have been fetched, calls were: " + adminOps.calls); + // listPartitionNames goes through the same funnel; a second entrance would be a way around it. + Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitionNames(null, handle)); + } + + /** + * A handle that names a partition column it carries no type for has come apart from the schema it + * was built from — an ALTER between the two reads, a handle assembled by hand. Assuming such a + * column readable would put exactly the names this guard exists to stop back on the path to + * fe-core's parser, so the unknown is refused and says which column it was. + */ + @Test + public void partitionColumnWithNoTypeIsRefusedRatherThanAssumed() { + FlussTableHandle handle = new FlussTableHandle("db", "part_table", 1L, 1, false, + Collections.emptyList(), Collections.emptyList(), 1, + Collections.singletonList("p"), false, null, + Collections.emptyMap(), Collections.emptyMap()); + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.STRING()); + FlussConnectorMetadata metadata = metadata(adminOps); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue(failure.getMessage().contains("'p'"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("no type"), failure.getMessage()); + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "the partitions should not have been fetched, calls were: " + adminOps.calls); + } + + @Test + public void binaryPartitionIsRefusedOnlyWhenTheColumnIsNotText() { + // Fluss names such a partition with the hex text of the bytes. Read as a string that is exactly + // what was written; asked for as a VARBINARY it is not a literal of anything. + FlussConnectorMetadata asText = metadata(withPartitionColumnOfType(DataTypes.BINARY(2))); + Assertions.assertEquals(1, asText.listPartitions(null, + asText.getTableHandle(null, "db", "part_table").orElseThrow(AssertionError::new), + Optional.empty()).size()); + + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.BINARY(2)); + FlussConnectorMetadata asVarbinary = metadata(adminOps, new FlussTypeMapping.Options(true, false)); + ConnectorTableHandle handle = asVarbinary.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> asVarbinary.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue( + failure.getMessage().contains(FlussConnectorProperties.ENABLE_MAPPING_VARBINARY), + "the fix is a property, so name it: " + failure.getMessage()); + } + @Test public void statisticsAreARowCountOrNothingAtAll() { RecordingFlussAdminOps adminOps = withLogTable(); diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java new file mode 100644 index 00000000000000..0982e7fb4302f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Which column types a fluss table may be partitioned by and still be readable. + * + *

The verdicts here were not reasoned out from fluss's naming rules and then trusted: every one of + * them was run against a fluss cluster through a Doris catalog first, and the split below is where the + * two sides actually part company. A type that renders into a partition name and back unchanged is + * allowed; one that fluss rewrites on the way in is refused, because the rewriting loses which character + * was there ({@code 1_5} was {@code 1.5}, {@code 01-02-03} was {@code 01:02:03}). + * + *

Completeness is the property worth more than any single verdict: every fluss type root is answered, + * so a type a future release adds to fluss's partition-key whitelist cannot arrive here as "allowed by + * omission" and reach fe-core's partition parser, where the failure names neither fluss nor the column. + */ +public class FlussPartitionColumnTypesTest { + + private static final FlussTypeMapping.Options VARBINARY = + new FlussTypeMapping.Options(true, false); + + /** One type per fluss type root, and the verdict each is meant to get as a PARTITION column. */ + private static Map verdicts() { + Map expected = new LinkedHashMap<>(); + // Stored in the partition name exactly as written, and read back the same way. + expected.put(DataTypes.CHAR(2), true); + expected.put(DataTypes.STRING(), true); + expected.put(DataTypes.BOOLEAN(), true); + expected.put(DataTypes.TINYINT(), true); + expected.put(DataTypes.SMALLINT(), true); + expected.put(DataTypes.INT(), true); + expected.put(DataTypes.BIGINT(), true); + expected.put(DataTypes.DATE(), true); + // Named with the hex text of the bytes, which is a string; readable as long as the catalog is + // mapping such a column to a string too. The other half of that pair is its own test below. + expected.put(DataTypes.BINARY(2), true); + expected.put(DataTypes.BYTES(), true); + // Fluss allows all five as partition keys and rewrites the '.', ':' and ' ' in their values. + expected.put(DataTypes.FLOAT(), false); + expected.put(DataTypes.DOUBLE(), false); + expected.put(DataTypes.TIME(), false); + expected.put(DataTypes.TIMESTAMP(3), false); + expected.put(DataTypes.TIMESTAMP_LTZ(3), false); + // Fluss refuses these as partition keys itself; refused here as well so that its refusal is not + // the only thing between them and fe-core's parser. + expected.put(DataTypes.DECIMAL(20, 4), false); + expected.put(DataTypes.ARRAY(DataTypes.INT()), false); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), false); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), false); + return expected; + } + + @Test + public void everyFlussTypeRootHasADeliberateVerdict() { + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : verdicts().entrySet()) { + DataType type = entry.getKey(); + covered.add(type.getTypeRoot()); + boolean readable = + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT) == null; + Assertions.assertEquals(entry.getValue(), readable, "partition verdict for " + type); + } + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "a fluss type root has no verdict here; decide whether a partition of it can be read"); + } + + /** + * The one verdict the catalog's own settings can flip. Fluss names a BINARY partition with the hex + * text of its bytes: readable while the column is a Doris string, unreadable the moment + * {@code enable.mapping.varbinary} turns it into a VARBINARY, which no hex text is a literal of. + */ + @Test + public void binaryPartitionsAreReadableOnlyWhileTheColumnIsText() { + for (DataType type : Arrays.asList(DataTypes.BINARY(2), DataTypes.BYTES())) { + Assertions.assertNull( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT), + type + " is readable while it maps to a string"); + String rejection = FlussPartitionColumnTypes.rejection(type, VARBINARY); + Assertions.assertNotNull(rejection, type + " under varbinary mapping"); + // The property that caused it is named, because turning it off is the fix and nothing else + // about the table changed. + Assertions.assertTrue( + rejection.contains(FlussConnectorProperties.ENABLE_MAPPING_VARBINARY), + "the rejection should name the property that caused it: " + rejection); + } + } + + /** + * The catalog's settings must not reach any other verdict. A property about binary columns deciding + * whether a DATE partition can be read would be a rule nobody could predict from its name. + */ + @Test + public void noOtherVerdictDependsOnTheCatalogSettings() { + for (DataType type : verdicts().keySet()) { + if (type.getTypeRoot() == DataTypeRoot.BINARY || type.getTypeRoot() == DataTypeRoot.BYTES) { + continue; + } + Assertions.assertEquals( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT) == null, + FlussPartitionColumnTypes.rejection(type, VARBINARY) == null, + "the verdict for " + type + " changed with the catalog's mapping options"); + } + } + + /** + * A refused type says why in words that name the loss, not just that something is unsupported: the + * reader of this error has a table fluss created happily and a DESC that looks ordinary. + */ + @Test + public void refusalsExplainWhatWasLost() { + List rewritten = Arrays.asList(DataTypes.FLOAT(), DataTypes.DOUBLE(), + DataTypes.TIME(), DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP_LTZ(3)); + for (DataType type : rewritten) { + String rejection = FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT); + Assertions.assertNotNull(rejection, type.toString()); + Assertions.assertTrue(rejection.contains("cannot be read back"), + "the rejection for " + type + " should say the value is lost, was: " + rejection); + } + } + + /** + * A timestamp is refused at every precision. Doris can hold the value at six digits or fewer, so a + * rule written around precision — the one the union-read key gate needs — would let those through + * and back into the parser that cannot read the name they arrive under. + */ + @Test + public void timestampsAreRefusedAtEveryPrecision() { + for (int precision = 0; precision <= 9; precision++) { + Assertions.assertNotNull( + FlussPartitionColumnTypes.rejection(DataTypes.TIMESTAMP(precision), + FlussTypeMapping.Options.DEFAULT), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNotNull( + FlussPartitionColumnTypes.rejection(DataTypes.TIMESTAMP_LTZ(precision), + FlussTypeMapping.Options.DEFAULT), + "TIMESTAMP_LTZ(" + precision + ")"); + } + } + + /** + * Readability is the weaker of the two questions asked about a partition column: whatever cannot be + * read at all cannot be matched to a lake split either. Were that ever the other way round, a table + * would be planned as a lake-plus-tail read on partition values nothing can parse. + */ + @Test + public void everythingReadableIsAlsoAskedTheStricterUnionQuestion() { + for (DataType type : verdicts().keySet()) { + if (FlussUnionKeyTypes.partitionColumnRejection(type) == null) { + Assertions.assertNull( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT), + type + " may be matched across the halves but cannot be read at all"); + } + } + } +} From 64b1fb81aae619f443d97c305d17438dbbff5471 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 4 Aug 2026 00:45:51 +0800 Subject: [PATCH 35/35] [test](fluss) Read every type, every partition type and a hundred thousand rows The seven suites so far each follow one path in depth, over fixtures small enough that a wrong answer is visible by reading it. Five things that leaves uncovered, and twelve fixtures for them: - Nesting. The type fixtures carry one column per fluss type and never more than one level, but an element decoder is chosen per level: being right about MAP says nothing about the MAP> beside it. log_nested, pk_nested and lake_nested carry every combination of the three constructors, three levels deep, through all three row formats -- arrow, compacted and parquet-through-paimon -- with rows that are NULL at the outer level and rows whose collections hold NULL elements, which is a different thing to get wrong. - Partition column types. Everything so far partitions by STRING, which is also the one type that cannot tell a rendering bug from a working one. part_types partitions by one column of every type whose value survives fluss's partition naming and prunes on each of them; part_ts is the counter-case, and lake_pk_part_int is the primary-key table whose halves must not be matched by a non-STRING partition value -- covering the fallback that until now only ever fired on tables with no lake at all. - Size. big_log and big_pk hold 100000 rows each, tiered, with a tail. A scan that loses its last partial batch, a stopping offset applied to one bucket, a suppression set built per split rather than per bucket: none of those can show up on nine rows, where wrong and right are the same arrangement. Each is read through a union-read catalog and a disabled one, so one fixture answers the log, primary-key and union-read cases and the two paths are compared against each other. Every column is derived from a sequence, so the aggregates are closed forms rather than transcriptions. - Compound predicates. A disjunction over a partition column has to keep every partition it names; a predicate that reached only one half of a union read returns an answer short by exactly the other half's matching rows, and a plausible one. Three-valued logic gets its own block. - The rest of what a catalog answers for: tables with nothing in them (pk_empty, lake_empty, and the aggregates that are 0 and NULL at once), the fluss TIME column Doris has nowhere to put, the ways a query is supposed to fail, and the ordinary SQL a user reaches for -- joins, subqueries, set operations, CTAS and insert-select into an internal table. Every recorded block was checked against the fixture's own literals, and each new baseline was perturbed by one value to confirm it fails when it should. The existing baselines change only by the new table names in two listings; no recorded data row moved. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016VCPjzhwMQuP7nTgdvGVbM --- .../docker-compose/fluss/README.md | 12 + .../fluss/scripts/run-init-sql.sh | 17 +- .../fluss/sql/init-lake-tail.sql | 68 ++++ .../docker-compose/fluss/sql/init.sql | 360 ++++++++++++++++++ .../fluss/sql/lake-row-counts.sql | 17 +- .../fluss/test_fluss_big_data.out | 98 +++++ .../fluss/test_fluss_catalog.out | 24 ++ .../fluss/test_fluss_lake_only.out | 12 + .../fluss/test_fluss_misc.out | 156 ++++++++ .../fluss/test_fluss_nested_types.out | 75 ++++ .../fluss/test_fluss_partition_types.out | 75 ++++ .../fluss/test_fluss_predicates.out | 157 ++++++++ .../fluss/test_fluss_big_data.groovy | 230 +++++++++++ .../fluss/test_fluss_misc.groovy | 261 +++++++++++++ .../fluss/test_fluss_nested_types.groovy | 194 ++++++++++ .../fluss/test_fluss_partition_types.groovy | 261 +++++++++++++ .../fluss/test_fluss_predicates.groovy | 222 +++++++++++ 17 files changed, 2235 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_big_data.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_misc.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out create mode 100644 regression-test/data/external_table_p0/fluss/test_fluss_predicates.out create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy create mode 100644 regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md index 38851863fcd805..249a4c6537badb 100644 --- a/docker/thirdparties/docker-compose/fluss/README.md +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -96,9 +96,15 @@ The fixtures recreate database `fluss_test` from scratch on every start: | `log_types` | log table, one column per mapped fluss type, plus an all-NULL row | | `log_part` | log table partitioned by `dt`, partitions `20260101`, `20260102`, `20260103` | | `log_empty` | log table with no rows at all (planning must emit zero scan ranges) | +| `log_nested` | log table whose complex types are nested inside complex types, plus rows with NULLs at every level | +| `log_time` | log table carrying a fluss TIME column, the one type Doris cannot represent | +| `part_types` | log table partitioned by one column of every type that survives fluss's partition naming (STRING, CHAR, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, DATE, BINARY) | +| `part_ts` | log table partitioned by a TIMESTAMP, whose value fluss rewrites into the partition name and nothing can read back | | `pk_basic` | primary-key table, one updated row and one deleted row | | `pk_types` | primary-key table with the same type coverage as `log_types` | | `pk_part` | primary-key table partitioned by `dt`, with an update and a delete inside a partition | +| `pk_nested` | primary-key table with the same nesting as `log_nested`, in the kv row format | +| `pk_empty` | primary-key table with no rows and therefore no kv snapshot | | `lake_log` | lake table, 4 rows tiered + 2 in the log, 3 buckets (some bucket has no tail) | | `lake_cold` | lake table read entirely from the lake — no log tail at all | | `lake_types` | lake table with the full type coverage; non-NULL rows tiered, the all-NULL row in the log | @@ -107,6 +113,12 @@ The fixtures recreate database `fluss_test` from scratch on every start: | `lake_pk_multi` | primary-key lake table over 3 buckets; the tail reaches some buckets and not others, which is what makes per-bucket binding observable | | `lake_pk_part` | primary-key lake table partitioned by `dt`: `20260101` is lake + tail, `20260102` is lake only, `20260103` was written after tiering stopped so the lake has never seen it | | `lake_pk_cold` | primary-key lake table read entirely from the lake — no tail, so nothing to merge | +| `lake_nested` | lake table with nested complex types; the populated row is tiered, the all-NULL row stays in the log | +| `lake_empty` | lake table nothing was ever written to, so tiering has never committed and there is no snapshot to read | +| `lake_part_int` | lake table partitioned by an INT; one partition has a tail, the other does not | +| `lake_pk_part_int` | primary-key lake table partitioned by an INT — the halves cannot be matched by a non-STRING partition value, so it falls back to the fluss-only read | +| `big_log` | lake table, 100000 rows tiered + 1000 in the log (ids 1..101000) | +| `big_pk` | primary-key lake table, 100000 keys tiered; the tail updates 500, adds 500 and deletes 5 | There is deliberately no deletion-vector fixture. Fluss does forward a `paimon.deletion-vectors.enabled` table property into the paimon table it diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh index e8b0c8fca59aa3..b06bef5f8e4032 100755 --- a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -46,8 +46,12 @@ SQL_TIMEOUT_SECONDS=900 ATTEMPTS=3 # Primary-key fixtures whose buckets must have been snapshotted before the # environment counts as ready. See wait_for_kv_snapshots. -SNAPSHOT_TABLES=(pk_basic pk_types pk_part lake_pk lake_pk_multi lake_pk_part lake_pk_cold) -SNAPSHOT_WAIT_SECONDS=120 +# +# pk_empty is deliberately absent: nothing was ever written to it, so no bucket +# will ever be snapshotted and waiting would hang on the state it exists to have. +SNAPSHOT_TABLES=(pk_basic pk_types pk_part pk_nested lake_pk lake_pk_multi lake_pk_part + lake_pk_cold lake_pk_part_int big_pk) +SNAPSHOT_WAIT_SECONDS=180 # What each lake fixture must hold in paimon before the tail is written -- the # row counts init.sql writes, merged where the table has a primary key. Keep in @@ -62,8 +66,15 @@ LAKE_EXPECTED_ROWS=( "lake_pk_multi=9" "lake_pk_part=4" "lake_pk_cold=3" + "lake_nested=1" + "lake_part_int=3" + "lake_pk_part_int=4" + "big_log=100000" + "big_pk=100000" ) -LAKE_TIERING_WAIT_SECONDS=300 +# The two large fixtures put 200000 rows through the tiering service, which is +# most of what this wait is now for; the small ones commit within a round. +LAKE_TIERING_WAIT_SECONDS=900 TIERING_JAR_GLOB='/opt/flink/opt/fluss-flink-tiering-*.jar' FLINK_BIN=/opt/flink/bin/flink diff --git a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql index 0d1dc09efc5469..cacb596b9c41b6 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql @@ -94,13 +94,81 @@ INSERT INTO lake_pk_part VALUES INSERT INTO lake_pk_part VALUES (5, 'pp3a', '20260103'); +-- The all-NULL row of the nested fixture: every nested column arrives from the +-- log half while its populated counterpart comes through paimon. +INSERT INTO lake_nested VALUES + ( + 2, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>) + ); + +-- A tail in one INT-named partition and none in the other. +INSERT INTO lake_part_int VALUES + (4, 'li1c', 1); + +-- A tail on the primary-key table whose partition column is an INT. It exists so +-- that falling back to the fluss-only read is a real choice rather than a +-- distinction without a difference: were the halves merged by key here, this row +-- would be the one the merge got wrong. +INSERT INTO lake_pk_part_int VALUES + (1, 'pi1a-hot', 1); + +-- The large tails. 500 updated keys, 500 new ones and five deletions, against a +-- lake of 100000 -- so the suppression set is a thousand-odd keys wide and the +-- rows it must drop are scattered through every bucket rather than sitting at +-- the front of one. +CREATE TEMPORARY TABLE tail_seq (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '1', + 'fields.id.end' = '500' +); + +CREATE TEMPORARY TABLE tail_seq_log (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '100001', + 'fields.id.end' = '101000' +); + +-- 1000 rows appended after the lake stopped: the log half of the big log table. +-- Ids stay contiguous with the lake half (1..101000), and every derived column +-- is the same function of the id on both sides, so an aggregate over the union +-- has a closed form and a missing or duplicated half is arithmetic, not opinion. +INSERT INTO big_log +SELECT id, CONCAT('hot', CAST(id AS STRING)), CAST(id AS DECIMAL(10, 2)), MOD(id, 7) +FROM tail_seq_log; + +-- 500 keys the lake already holds, written again with a different name: each one +-- must suppress exactly the lake row it replaces. +INSERT INTO big_pk +SELECT id, CONCAT('hot', CAST(id AS STRING)), MOD(id, 7) FROM tail_seq; + +-- 500 keys the lake has never seen, in buckets it does hold rows for: they are +-- added, not substituted for anything. +INSERT INTO big_pk +SELECT 100000 + id, CONCAT('new', CAST(id AS STRING)), MOD(100000 + id, 7) FROM tail_seq; + -- Deletions of rows the lake holds. They are the case a suppression set has to -- cover but a merge of surviving rows cannot: the key is gone from the tail's -- own result, and the lake row it removes would otherwise stay. +-- +-- The big table's five are spelled out one at a time because a fluss DELETE +-- resolves a full primary key, not a range; they are spread across the key space +-- so they cannot all land in one bucket. SET 'execution.runtime-mode' = 'batch'; DELETE FROM lake_pk WHERE id = 1; DELETE FROM lake_pk_multi WHERE id = 7; DELETE FROM lake_pk_part WHERE id = 2 AND dt = '20260101'; +DELETE FROM big_pk WHERE id = 7; +DELETE FROM big_pk WHERE id = 19999; +DELETE FROM big_pk WHERE id = 50000; +DELETE FROM big_pk WHERE id = 77777; +DELETE FROM big_pk WHERE id = 99999; SET 'execution.runtime-mode' = 'streaming'; -- lake_pk_cold gets nothing: it is the fixture for a primary-key table the lake diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql index 4245c7a7797277..a01b4781d30d7b 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/init.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -183,6 +183,187 @@ CREATE TABLE log_empty ( 'bucket.num' = '2' ); +-- --------------------------------------------------------------------------- +-- log_nested: complex types nested inside complex types. log_types covers one +-- column per type but never more than one level, and a decoder can be right +-- about a MAP and wrong about the MAP> beside it -- +-- the element decoder is chosen per level, and only a nested column asks for it +-- to be chosen twice. Every combination of the three constructors is here, plus +-- an all-NULL row, because a null at the outer level and a null element inside a +-- present collection are different things to get wrong. +-- --------------------------------------------------------------------------- +CREATE TABLE log_nested ( + id INT, + f_arr_arr ARRAY>, + f_arr_map ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW>, + f_arr_arr_arr ARRAY>> +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[MAP['a', 1], MAP['b', 2]], + ARRAY[CAST(ROW(1, 'x') AS ROW), CAST(ROW(2, 'y') AS ROW)], + MAP['k1', ARRAY[1, 2], 'k2', ARRAY[3]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>), + ARRAY[ARRAY[ARRAY[1], ARRAY[2, 3]], ARRAY[ARRAY[4]]] + ), + ( + 2, + ARRAY[CAST(NULL AS ARRAY)], + ARRAY[CAST(NULL AS MAP)], + ARRAY[CAST(NULL AS ROW)], + MAP['k1', CAST(NULL AS ARRAY)], + MAP['k1', CAST(NULL AS ROW)], + CAST(ROW(CAST(NULL AS INT), CAST(NULL AS ARRAY), CAST(NULL AS MAP), + CAST(NULL AS ROW)) + AS ROW, r_map MAP, r_row ROW>), + ARRAY[CAST(NULL AS ARRAY>)] + ), + ( + 3, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>), + CAST(NULL AS ARRAY>>) + ); + +-- --------------------------------------------------------------------------- +-- pk_nested: the same nesting in the kv row format a primary-key table stores. +-- The two formats have separate readers for every constructor, so covering one +-- says nothing about the other. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_nested ( + id INT NOT NULL, + f_arr_arr ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW>, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO pk_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[CAST(ROW(1, 'x') AS ROW)], + MAP['k1', ARRAY[1, 2]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>) + ), + ( + 2, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>) + ); + +-- The update is what makes this a primary-key fixture rather than a second log +-- one: the nested values of key 1 have to be the SECOND set, not the first. +INSERT INTO pk_nested VALUES + ( + 1, + ARRAY[ARRAY[10, 20]], + ARRAY[CAST(ROW(11, 'xx') AS ROW)], + MAP['k9', ARRAY[10]], + MAP['k9', CAST(ROW(99, 'zz') AS ROW)], + CAST(ROW(2, ARRAY[70], MAP['mm', 30], ROW(50, 'deeper')) + AS ROW, r_map MAP, r_row ROW>) + ); + +-- --------------------------------------------------------------------------- +-- part_types: one partition column of every type fluss allows AND Doris can +-- read back. A partition's value is kept nowhere but in its name, so the type +-- decides whether it survives at all -- STRING is the only one the rest of these +-- fixtures use, and it is also the only one that could not tell a rendering bug +-- from a working one. +-- +-- Two rows, two partitions: one value of each column per partition, so a +-- predicate on any single column prunes to exactly one. +-- +-- p_bin is the type whose verdict depends on the catalog: fluss names its +-- partitions with the hex text of the bytes, which reads back as the text it is +-- unless 'enable.mapping.varbinary' asks for a VARBINARY column instead. +-- --------------------------------------------------------------------------- +CREATE TABLE part_types ( + id INT, + name STRING, + p_str STRING, + p_char CHAR(2), + p_bool BOOLEAN, + p_tiny TINYINT, + p_small SMALLINT, + p_int INT, + p_big BIGINT, + p_date DATE, + p_bin BINARY(2) +) PARTITIONED BY (p_str, p_char, p_bool, p_tiny, p_small, p_int, p_big, p_date, p_bin) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO part_types VALUES + (1, 'pt1', 'cn', CAST('c1' AS CHAR(2)), TRUE, CAST(1 AS TINYINT), CAST(10 AS SMALLINT), 100, + CAST(1000 AS BIGINT), DATE '2026-01-01', CAST(X'0102' AS BINARY(2))), + (2, 'pt2', 'us', CAST('c2' AS CHAR(2)), FALSE, CAST(2 AS TINYINT), CAST(20 AS SMALLINT), 200, + CAST(2000 AS BIGINT), DATE '2026-01-02', CAST(X'0304' AS BINARY(2))); + +-- --------------------------------------------------------------------------- +-- part_ts: a partition column whose value fluss cannot store verbatim. Its name +-- for 2026-01-01 01:02:03 is 2026-01-01-01-02-03_0 -- every character a +-- partition name may not hold rewritten, many-to-one, unrecoverable. Fluss +-- creates the table without complaint, which is why the refusal has to come from +-- the connector and has to name the column. +-- --------------------------------------------------------------------------- +CREATE TABLE part_ts ( + id INT, + name STRING, + p_ts TIMESTAMP(3) +) PARTITIONED BY (p_ts) +WITH ( + 'bucket.num' = '1' +); + +INSERT INTO part_ts VALUES + (1, 'ts1', TIMESTAMP '2026-01-01 01:02:03.000'); + +-- --------------------------------------------------------------------------- +-- log_time: a column of the one fluss type Doris has nowhere to put. The column +-- reads as UNSUPPORTED, so naming it -- or asking for * -- must fail, while the +-- rest of the table stays perfectly readable. A connector that mapped it to a +-- string or to elapsed millis instead would hand back a value meaning something +-- else, which no error would ever reveal. +-- --------------------------------------------------------------------------- +CREATE TABLE log_time ( + id INT, + name STRING, + f_time TIME(0) +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_time VALUES + (1, 'time1', TIME '01:02:03'), + (2, 'time2', TIME '04:05:06'); + -- --------------------------------------------------------------------------- -- pk_basic: primary-key table. Row 2 is updated and row 3 deleted, so a -- correct read returns the merged view, not the raw change log. @@ -312,6 +493,20 @@ SET 'execution.runtime-mode' = 'batch'; DELETE FROM pk_part WHERE id = 4 AND dt = '20260102'; SET 'execution.runtime-mode' = 'streaming'; +-- --------------------------------------------------------------------------- +-- pk_empty: a primary-key table nothing was ever written to. It is not the same +-- shape as an empty log table: a primary-key read starts from a kv snapshot, +-- and there is none, so the path taken here is the one that has to notice the +-- table is empty rather than the one that reads no log records. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_empty ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '2' +); + -- =========================================================================== -- Lake tables. 'table.datalake.enabled' makes the fluss coordinator create a -- matching paimon table and lets the tiering service move data into it; the @@ -556,6 +751,171 @@ INSERT INTO lake_pk_cold VALUES (2, 'c2'), (3, 'c3'); +-- --------------------------------------------------------------------------- +-- lake_nested: nested complex types that cross the seam. The type mapping this +-- connector applies has to equal fluss->paimon->Doris for these as well, and +-- nesting is where the two could most easily part ways: each level is converted +-- by its own rule on both sides. The populated row is tiered, the all-NULL row +-- stays in the log, so one union read decodes the same nested column through +-- paimon and through fluss. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_nested ( + id INT, + f_arr_arr ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW> +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[CAST(ROW(1, 'x') AS ROW)], + MAP['k1', ARRAY[1, 2]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>) + ); + +-- --------------------------------------------------------------------------- +-- lake_empty: tiering is on and has never committed anything, because nothing +-- was ever written. It is the state every lake table passes through, and the +-- one where "read the lake plus the log" has no lake to read: auto has to fall +-- back to the fluss-only read, required has to say why it will not. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_empty ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +-- --------------------------------------------------------------------------- +-- lake_part_int: a tiered log table partitioned by an INT. Concatenating a lake +-- with a log tail needs no partition value matched across the halves -- each +-- half prunes on its own -- so a non-STRING partition column has to work here, +-- and this table is what says the rule that stops the primary-key merge (below) +-- was not applied to everything partitioned. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_part_int ( + id INT, + name STRING, + p_int INT +) PARTITIONED BY (p_int) +WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_part_int VALUES + (1, 'li1a', 1), + (2, 'li1b', 1), + (3, 'li2a', 2); + +-- --------------------------------------------------------------------------- +-- lake_pk_part_int: a tiered PRIMARY-KEY table partitioned by an INT. Merging +-- its halves by key means matching a paimon split to a fluss partition by the +-- text each side renders that value as, and only STRING is guaranteed to render +-- alike -- so this table must NOT be merged. Under auto it falls back to the +-- fluss-only read, which returns every row anyway; under required it is an +-- error. Without this fixture the rule is only ever exercised on tables that had +-- no lake to merge in the first place. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_part_int ( + id INT NOT NULL, + name STRING, + p_int INT NOT NULL, + PRIMARY KEY (id, p_int) NOT ENFORCED +) PARTITIONED BY (p_int) +WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_part_int VALUES + (1, 'pi1a', 1), + (2, 'pi1b', 1), + (3, 'pi2a', 2), + (4, 'pi2b', 2); + +-- =========================================================================== +-- The large fixtures. Everything above is a handful of rows chosen so that a +-- wrong answer is visible by eye; these two are the opposite question -- whether +-- the same machinery still returns the right answer when a scan is split across +-- many batches, a suppression set holds more keys than a debugger would print, +-- and a bucket's log runs to five figures. +-- +-- Both are lake tables with a tail, so ONE table answers three of the scenarios +-- at once: read through a union-read catalog it is lake plus log, read through a +-- disabled one it is the pure fluss read of the same rows -- a full log replay +-- for big_log, a kv snapshot plus its log for big_pk. Two catalogs over one +-- fixture also means the two paths can be compared to each other rather than to +-- a number someone wrote down. +-- +-- The values are derived from the sequence rather than randomly generated, so +-- every aggregate over them is a closed form: 1..100000 sums to 5000050000, and +-- a suite asserting that is checking arithmetic rather than repeating whatever +-- the fixture happened to produce. +-- =========================================================================== + +CREATE TEMPORARY TABLE big_seq (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '1', + 'fields.id.end' = '100000' +); + +-- --------------------------------------------------------------------------- +-- big_log: 100000 rows over three buckets, all of them tiered. The tail written +-- afterwards is small on purpose -- that is the shape of a real tiered table, +-- where the tail is only what the freshness window has not yet taken. +-- --------------------------------------------------------------------------- +CREATE TABLE big_log ( + id INT, + name STRING, + price DECIMAL(10, 2), + grp INT +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO big_log +SELECT id, CONCAT('n', CAST(id AS STRING)), CAST(id AS DECIMAL(10, 2)), MOD(id, 7) FROM big_seq; + +-- --------------------------------------------------------------------------- +-- big_pk: 100000 keys over three buckets, all tiered. Its tail updates 500 of +-- them, deletes five and adds 500 more, so the suppression set that filters the +-- lake half holds a thousand-odd keys instead of the two or three every other +-- primary-key fixture here has. A merge that is right for three keys and wrong +-- for a thousand -- a set built per split, a cache keyed too coarsely -- has +-- nowhere to hide in the counts this produces. +-- --------------------------------------------------------------------------- +CREATE TABLE big_pk ( + id INT NOT NULL, + name STRING, + grp INT, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO big_pk +SELECT id, CONCAT('p', CAST(id AS STRING)), MOD(id, 7) FROM big_seq; + -- No deletion-vector fixture, and the reason is worth recording where the next -- person to want one will look. Fluss does forward a 'paimon.*' table property -- into the paimon table it creates, so a table declared with diff --git a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql index d1ab414d1623a2..53d8772c86ce6f 100644 --- a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql +++ b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql @@ -55,4 +55,19 @@ SELECT CONCAT('LAKEROWS:lake_pk_multi=', CAST(COUNT(*) AS STRING)) FROM lake_pk_ UNION ALL SELECT CONCAT('LAKEROWS:lake_pk_part=', CAST(COUNT(*) AS STRING)) FROM lake_pk_part UNION ALL -SELECT CONCAT('LAKEROWS:lake_pk_cold=', CAST(COUNT(*) AS STRING)) FROM lake_pk_cold; +SELECT CONCAT('LAKEROWS:lake_pk_cold=', CAST(COUNT(*) AS STRING)) FROM lake_pk_cold +UNION ALL +SELECT CONCAT('LAKEROWS:lake_nested=', CAST(COUNT(*) AS STRING)) FROM lake_nested +UNION ALL +SELECT CONCAT('LAKEROWS:lake_part_int=', CAST(COUNT(*) AS STRING)) FROM lake_part_int +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_part_int=', CAST(COUNT(*) AS STRING)) FROM lake_pk_part_int +UNION ALL +SELECT CONCAT('LAKEROWS:big_log=', CAST(COUNT(*) AS STRING)) FROM big_log +UNION ALL +SELECT CONCAT('LAKEROWS:big_pk=', CAST(COUNT(*) AS STRING)) FROM big_pk; + +-- lake_empty is absent on purpose: nothing is ever written to it, so there is +-- nothing for tiering to commit and no count to wait for. It is the fixture for +-- a lake table that has no snapshot at all, and waiting for one would hang the +-- environment on the very state it exists to provide. diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out b/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out new file mode 100644 index 00000000000000..4b107a55b0676a --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out @@ -0,0 +1,98 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !log_summary -- +101000 5100550500 1 101000 7 5100550500.00 + +-- !log_distinct -- +101000 + +-- !log_groups -- +0 14428 +1 14429 +2 14429 +3 14429 +4 14429 +5 14428 +6 14428 + +-- !log_seam -- +1 n1 1.00 1 +100000 n100000 100000.00 5 +100001 hot100001 100001.00 6 +100002 hot100002 100002.00 0 +100999 hot100999 100999.00 3 +101000 hot101000 101000.00 4 +2 n2 2.00 2 +50000 n50000 50000.00 6 +99999 n99999 99999.00 4 + +-- !log_range -- +21 2100000 + +-- !log_projection -- +1000 + +-- !log_ordered_head -- +1 n1 +2 n2 +3 n3 +4 n4 +5 n5 + +-- !log_ordered_tail -- +100996 hot100996 +100997 hot100997 +100998 hot100998 +100999 hot100999 +101000 hot101000 + +-- !pk_summary -- +100495 5049927468 1 100500 7 + +-- !pk_distinct -- +100495 + +-- !pk_by_origin -- +499 500 99496 + +-- !pk_deleted -- + +-- !pk_neighbours -- +100000 p100000 +19998 p19998 +20000 p20000 +49999 p49999 +50001 p50001 +6 hot6 +8 hot8 +99998 p99998 + +-- !pk_sample -- +1 hot1 1 +100000 p100000 5 +100001 new1 6 +100500 new500 1 +500 hot500 3 +501 p501 4 + +-- !pk_groups -- +0 14354 +1 14358 +2 14357 +3 14357 +4 14356 +5 14357 +6 14356 + +-- !pk_ordered_head -- +1 hot1 +2 hot2 +3 hot3 +4 hot4 +5 hot5 + +-- !fluss_only_log -- +101000 5100550500 101000 + +-- !fluss_only_pk -- +100495 5049927468 100500 + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out index fe067cfb830b31..9afbb0b41657a1 100644 --- a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out +++ b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out @@ -6,19 +6,31 @@ information_schema mysql -- !tables -- +big_log +big_pk lake_cold +lake_empty lake_log +lake_nested lake_part +lake_part_int lake_pk lake_pk_cold lake_pk_multi lake_pk_part +lake_pk_part_int lake_types log_basic log_empty +log_nested log_part +log_time log_types +part_ts +part_types pk_basic +pk_empty +pk_nested pk_part pk_types @@ -59,19 +71,31 @@ name text Yes true \N score double Yes true \N -- !tables_after_refresh -- +big_log +big_pk lake_cold +lake_empty lake_log +lake_nested lake_part +lake_part_int lake_pk lake_pk_cold lake_pk_multi lake_pk_part +lake_pk_part_int lake_types log_basic log_empty +log_nested log_part +log_time log_types +part_ts +part_types pk_basic +pk_empty +pk_nested pk_part pk_types diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out index 2dd86d2b8b6136..18b6c9051a6a09 100644 --- a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out @@ -102,19 +102,31 @@ lake3 lake4 -- !tables -- +big_log +big_pk lake_cold +lake_empty lake_log +lake_nested lake_part +lake_part_int lake_pk lake_pk_cold lake_pk_multi lake_pk_part +lake_pk_part_int lake_types log_basic log_empty +log_nested log_part +log_time log_types +part_ts +part_types pk_basic +pk_empty +pk_nested pk_part pk_types diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_misc.out b/regression-test/data/external_table_p0/fluss/test_fluss_misc.out new file mode 100644 index 00000000000000..e742514350e909 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_misc.out @@ -0,0 +1,156 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pk_empty_count -- +0 + +-- !pk_empty_rows -- + +-- !empty_aggregates -- +0 0 \N \N \N \N + +-- !log_empty_aggregates -- +0 0 \N \N \N + +-- !empty_join -- +0 + +-- !lake_empty_count -- +0 + +-- !desc_log_time -- +id int Yes true \N +name text Yes true \N +f_time unknown type: UNSUPPORTED_TYPE Yes true \N + +-- !time_other_columns -- +1 time1 +2 time2 + +-- !time_count -- +2 + +-- !join_fluss -- +1 alice string1 +2 bob string2 +3 carol \N + +-- !join_pk -- +1 k1 alice +2 k2-updated bob + +-- !join_lake -- +1 lake1 cold1 +2 lake2 cold2 +3 lake3 cold3 +4 lake4 \N +5 hot5 \N +6 hot6 \N + +-- !join_self -- +1 p1a p1b + +-- !subquery_in -- +2 bob + +-- !subquery_exists -- +1 +2 +3 + +-- !subquery_scalar -- +2 bob +3 carol + +-- !cte -- +2 5 + +-- !union_all -- +1 alice +1 cold1 +2 bob +2 cold2 +3 carol +3 cold3 + +-- !union_distinct -- +1 +2 +3 + +-- !except -- +4 +5 +6 + +-- !intersect -- +1 +2 +3 + +-- !aggregates -- +3 3 10.10 30.30 60.60 2 + +-- !group_having -- +20260101 2 + +-- !order_limit -- +2 bob +3 carol + +-- !limit_offset -- +2 + +-- !star -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !expression -- +2 ALICE +4 BOB + +-- !case -- +1 low +2 high +3 high + +-- !copied -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !desc_ctas -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes false \N NONE +f_int int Yes false \N NONE +f_bigint bigint Yes false \N NONE +f_float float Yes false \N NONE +f_double double Yes false \N NONE +f_decimal decimal(20,4) Yes false \N NONE +f_char text Yes false \N NONE +f_string text Yes false \N NONE +f_date date Yes false \N NONE +f_timestamp datetime(6) Yes false \N NONE +f_timestamp_ltz datetime(3) Yes false \N NONE +f_array array Yes false \N NONE +f_map map Yes false \N NONE +f_row struct Yes false \N NONE + +-- !ctas_count -- +3 2 2 + +-- !desc_ctas_nested -- +id int Yes true \N +f_arr_arr array> Yes false \N NONE +f_map_row map> Yes false \N NONE +f_row_deep struct,r_map:map,r_row:struct> Yes false \N NONE + +-- !ctas_nested_rows -- +1 [[1, 2], [3]] {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} + +-- !join_internal -- +1 alice string1 +2 bob string2 +3 carol \N + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out b/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out new file mode 100644 index 00000000000000..50f890722b8a0f --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_log_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_map array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N +f_arr_arr_arr array>> Yes true \N + +-- !desc_pk_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N + +-- !log_rows -- +1 [[1, 2], [3]] [{"a":1}, {"b":2}] [{"a":1, "b":"x"}, {"a":2, "b":"y"}] {"k2":[3], "k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} [[[1], [2, 3]], [[4]]] +2 [null] [null] [null] {"k1":null} {"k1":null} {"r_int":null, "r_arr":null, "r_map":null, "r_row":null} [null] +3 \N \N \N \N \N \N \N + +-- !log_elements -- +1 2 [1, 2] 2 2 1 1 x [1, 2] 2 9 z +2 1 \N \N \N \N \N \N \N \N \N \N +3 \N \N \N \N \N \N \N \N \N \N \N + +-- !log_deep -- +1 2 1 8 3 5 deep +2 \N \N \N \N \N \N +3 \N \N \N \N \N \N + +-- !log_nested_predicate -- +1 + +-- !pk_rows -- +1 [[10, 20]] [{"a":11, "b":"xx"}] {"k9":[10]} {"k9":{"a":99, "b":"zz"}} {"r_int":2, "r_arr":[70], "r_map":{"mm":30}, "r_row":{"x":50, "y":"deeper"}} +2 \N \N \N \N \N + +-- !pk_elements -- +1 1 10 xx [10] 10 99 70 deeper +2 \N \N \N \N \N \N \N \N + +-- !desc_lake_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N + +-- !desc_lake_nested_lake -- +id int Yes true \N NONE +f_arr_arr array> Yes true \N NONE +f_arr_row array> Yes true \N NONE +f_map_arr map> Yes true \N NONE +f_map_row map> Yes true \N NONE +f_row_deep struct,r_map:map,r_row:struct> Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !lake_side -- +1 [[1, 2], [3]] [{"a":1, "b":"x"}] {"k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} + +-- !lake_union -- +1 [[1, 2], [3]] [{"a":1, "b":"x"}] {"k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} +2 \N \N \N \N \N + +-- !lake_union_elements -- +1 2 x 1 z deep +2 \N \N \N \N \N + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out b/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out new file mode 100644 index 00000000000000..71237c385fe989 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_part_types -- +id int Yes true \N +name text Yes true \N +p_str text Yes true \N +p_char char(2) Yes true \N +p_bool boolean Yes true \N +p_tiny tinyint Yes true \N +p_small smallint Yes true \N +p_int int Yes true \N +p_big bigint Yes true \N +p_date date Yes true \N +p_bin text Yes true \N + +-- !part_names -- +p_str=cn/p_char=c1/p_bool=true/p_tiny=1/p_small=10/p_int=100/p_big=1000/p_date=2026-01-01/p_bin=0102 +p_str=us/p_char=c2/p_bool=false/p_tiny=2/p_small=20/p_int=200/p_big=2000/p_date=2026-01-02/p_bin=0304 + +-- !part_rows -- +1 pt1 cn c1 true 1 10 100 1000 2026-01-01 0102 30313032 +2 pt2 us c2 false 2 20 200 2000 2026-01-02 0304 30333034 + +-- !pruned_str -- +1 pt1 + +-- !pruned_int -- +2 pt2 + +-- !pruned_date -- +2 pt2 + +-- !pruned_bool -- +2 pt2 + +-- !pruned_tiny -- +2 pt2 + +-- !pruned_multi -- +1 pt1 + +-- !pruned_impossible -- +0 + +-- !pruned_absent -- +0 + +-- !desc_part_ts -- +id int Yes true \N +name text Yes true \N +p_ts datetime(3) Yes true \N + +-- !lake_part_int -- +1 li1a 1 +2 li1b 1 +3 li2a 2 +4 li1c 1 + +-- !lake_part_int_tail -- +1 li1a +2 li1b +4 li1c + +-- !lake_part_int_tiered -- +3 li2a + +-- !lake_pk_part_int -- +1 pi1a-hot 1 +2 pi1b 1 +3 pi2a 2 +4 pi2b 2 + +-- !lake_pk_part_int_pruned -- +1 pi1a-hot +2 pi1b + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out b/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out new file mode 100644 index 00000000000000..b7bec6243de213 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out @@ -0,0 +1,157 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !and -- +2 bob + +-- !or -- +1 alice +3 carol + +-- !and_or -- +1 alice +3 carol + +-- !or_and -- +3 carol + +-- !not -- +2 bob + +-- !not_between -- +1 + +-- !in -- +1 alice + +-- !not_in -- +3 + +-- !like -- +2 + +-- !always_false -- +0 + +-- !always_true -- +3 + +-- !is_null -- +3 + +-- !is_not_null -- +1 +2 + +-- !ne_excludes_null -- +2 + +-- !not_eq_excludes_null -- +2 + +-- !null_or -- +1 +3 + +-- !null_and -- +3 + +-- !mixed_types -- +1 + +-- !mixed_types_or -- +1 +2 +3 + +-- !nested_elements -- +1 + +-- !part_or -- +1 p1a 20260101 +2 p1b 20260101 +4 p3a 20260103 + +-- !part_in -- +2 20260101 +3 20260102 + +-- !part_ne -- +3 20260102 +4 20260103 + +-- !part_and_data -- +2 p1b + +-- !part_or_data -- +1 p1a 20260101 +2 p1b 20260101 +4 p3a 20260103 + +-- !part_absent -- +0 + +-- !pk_and -- +2 k2-updated +4 k4 + +-- !pk_or -- +1 k1 +2 k2-updated + +-- !pk_stale_value -- +0 + +-- !pk_not -- +2 k2-updated +4 k4 + +-- !pk_in -- +1 1.5 +2 22.5 + +-- !union_and -- +3 lake3 +4 lake4 +5 hot5 + +-- !union_or -- +1 lake1 +6 hot6 + +-- !union_not -- +1 +6 + +-- !union_like -- +1 lake1 +5 hot5 +6 hot6 + +-- !union_lake_only -- +1 lake1 +2 lake2 +3 lake3 +4 lake4 + +-- !union_log_only -- +5 hot5 +6 hot6 + +-- !union_neither -- +0 + +-- !union_part_or -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 +4 lp1c 20260101 + +-- !union_part_and -- +4 + +-- !union_pk_and -- +3 lp3-hot +4 lp4-hot + +-- !union_pk_or -- +4 lp4-hot + diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy new file mode 100644 index 00000000000000..fd75a2bc56d1e9 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The same reads, at a size where nothing can be eyeballed. +// +// Every other fluss suite here works on a handful of rows chosen so that a wrong +// answer is visible by reading it. That leaves a whole class of faults uncovered: +// a scan that loses the last partial batch, a stopping offset applied to the first +// bucket only, a suppression set built per split rather than per bucket, a cache +// keyed too coarsely. None of them can show up on nine rows -- with one batch and +// one bucket's worth of keys, wrong and right are the same arrangement. +// +// Two fixtures of 100000 rows, each read down both paths: through a union-read +// catalog they are lake plus log, through a disabled one they are the pure fluss +// read of the same rows -- a full log replay for big_log, a kv snapshot plus its +// log for big_pk. So one table answers the log, primary-key and union-read cases, +// and the two paths can be compared against each other rather than against a +// number someone wrote down. +// +// What IS written down is arithmetic: the fixtures derive every column from a +// sequence, so 1..101000 sums to 5100550500 and nothing here is a transcription of +// whatever the environment happened to produce. +// +// Fixtures: big_log and big_pk in +// docker/thirdparties/docker-compose/fluss/sql/init.sql and init-lake-tail.sql. +suite("test_fluss_big_data", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String unionCatalog = "test_fluss_big_data" + String flussOnlyCatalog = "test_fluss_big_data_off" + + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + // The load-bearing check of this suite. At this size the recorded aggregates say a + // number came back; only the comparison says the OTHER reader agreed with it, and + // the two readers here share no code below the scan node. + def compareModes = { String query -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + } + + // --- a log table of 101000 rows ------------------------------------------- + // 100000 tiered plus 1000 written after, with contiguous ids, so the count, the + // sum and the bounds are all closed forms: 101000 rows, sum 5100550500, ids 1 to + // 101000. A seam that dropped the tail, read it twice or stopped a bucket early + // moves at least one of those. + def logPlan = planOf("""select * from big_log""") + assertTrue(logPlan.contains("unionRead=yes"), "not a union read: ${logPlan}") + assertTrue(countIn(logPlan, "lakeSplits") >= 1, "no lake splits: ${logPlan}") + def logRanges = countIn(logPlan, "logRanges") + assertTrue(logRanges >= 1 && logRanges <= 3, + "big_log planned ${logRanges} log ranges over 3 buckets: ${logPlan}") + + order_qt_log_summary """ + select count(*) as rows_read, sum(id) as id_sum, min(id) as id_min, max(id) as id_max, + count(distinct grp) as groups, sum(price) as price_sum + from big_log + """ + compareModes("""select count(*), sum(id), min(id), max(id), sum(price) from big_log""") + + // Every id exactly once. count(distinct) over 101000 values is what says so, and it + // is a different question from the count: a row read twice keeps the count wrong and + // the distinct count right, and one lost key does the opposite. + order_qt_log_distinct """select count(distinct id) from big_log""" + + // Per-group counts, which is the whole scan grouped rather than aggregated flat: a + // half silently dropped shows up as one group short of its share, not as zero. + order_qt_log_groups """select grp, count(*) from big_log group by grp order by grp""" + compareModes("""select grp, count(*) from big_log group by grp order by grp""") + + // Rows from both halves by name, including the two ids either side of the seam. + order_qt_log_seam """ + select id, name, price, grp from big_log + where id in (1, 2, 50000, 99999, 100000, 100001, 100002, 100999, 101000) + """ + + // A range predicate that spans the seam: the rows it selects come from the lake up + // to 100000 and from the log after it, in one result. + order_qt_log_range """select count(*), sum(id) from big_log where id between 99990 and 100010""" + order_qt_log_projection """select count(*) from big_log where name like 'hot%'""" + order_qt_log_ordered_head """select id, name from big_log order by id limit 5""" + order_qt_log_ordered_tail """select id, name from big_log order by id desc limit 5""" + + // --- a primary-key table of 100495 rows ----------------------------------- + // 100000 keys in the lake; the tail updates 500 of them, adds 500 more and deletes + // five. So the suppression set that filters the lake half holds 1005 keys rather + // than the two or three every other primary-key fixture has, and the rows it must + // drop are scattered through every bucket instead of sitting at the front of one. + // + // 100000 - 5 + 500 = 100495 rows. Sum of the surviving ids is + // 5000050000 - (7 + 19999 + 50000 + 77777 + 99999) + sum(100001..100500) = + // 5000050000 - 247782 + 50125250 = 5049927468. + def pkPlan = planOf("""select * from big_pk""") + assertTrue(pkPlan.contains("unionRead=yes"), "not a union read: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "logRanges"), + "a primary-key table planned a plain log range: ${pkPlan}") + def lakeSplits = countIn(pkPlan, "lakeSplits") + def suppressed = countIn(pkPlan, "suppressedLakeSplits") + assertTrue(lakeSplits >= 3, "three buckets should each contribute a lake split: ${pkPlan}") + // 500 updated keys hash across all three buckets, so every lake split has a tail to + // be filtered by and none may be let through unfiltered. + assertEquals(lakeSplits, suppressed, + "every bucket has a tail, so every lake split must be suppressed: ${pkPlan}") + assertEquals(3, countIn(pkPlan, "pkTailRanges"), "one tail per bucket: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "pkRanges"), + "the lake holds every bucket, so none is read whole: ${pkPlan}") + + order_qt_pk_summary """ + select count(*) as rows_read, sum(id) as id_sum, min(id) as id_min, max(id) as id_max, + count(distinct grp) as groups + from big_pk + """ + compareModes("""select count(*), sum(id), min(id), max(id) from big_pk""") + order_qt_pk_distinct """select count(distinct id) from big_pk""" + + // What the merge had to get right, counted by which half the winning row came from: + // 500 keys the tail replaced (less the one it then deleted), 500 it added, and the + // rest untouched in the lake. A suppression set that missed keys leaves the 'p' + // count too high and the total wrong by the same amount. + order_qt_pk_by_origin """ + select + sum(case when name like 'hot%' then 1 else 0 end) as replaced, + sum(case when name like 'new%' then 1 else 0 end) as added, + sum(case when name like 'p%' then 1 else 0 end) as untouched + from big_pk + """ + compareModes(""" + select + sum(case when name like 'hot%' then 1 else 0 end), + sum(case when name like 'new%' then 1 else 0 end), + sum(case when name like 'p%' then 1 else 0 end) + from big_pk + """) + + // The five deleted keys, one at a time. A delete is the case a merge of surviving + // rows cannot express: the key is absent from the tail's own result, and the lake + // row it removes stays unless something drops it. + order_qt_pk_deleted """ + select id, name from big_pk where id in (7, 19999, 50000, 77777, 99999) + """ + // And the keys either side of them, so "returns nothing" cannot be how the rows + // above came to be missing. + order_qt_pk_neighbours """ + select id, name from big_pk where id in (6, 8, 19998, 20000, 49999, 50001, 99998, 100000) + """ + + // Updated, added and untouched keys by name. + order_qt_pk_sample """ + select id, name, grp from big_pk + where id in (1, 500, 501, 99999, 100000, 100001, 100500) + """ + order_qt_pk_groups """select grp, count(*) from big_pk group by grp order by grp""" + order_qt_pk_ordered_head """select id, name from big_pk order by id limit 5""" + + // --- the same tables with the lake switched off --------------------------- + // A full change-log replay of 101000 records for the log table, and a kv snapshot + // plus its log for the primary-key one. These are the reads the other suites only + // ever ask for a few rows of. + sql """switch ${flussOnlyCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def flussOnlyLogPlan = planOf("""select * from big_log""") + assertTrue(flussOnlyLogPlan.contains("unionRead=no"), "the lake should be off: ${flussOnlyLogPlan}") + assertEquals(0, countIn(flussOnlyLogPlan, "lakeSplits"), "no lake half: ${flussOnlyLogPlan}") + order_qt_fluss_only_log """ + select count(*) as rows_read, sum(id) as id_sum, max(id) as id_max from big_log + """ + + def flussOnlyPkPlan = planOf("""select * from big_pk""") + assertTrue(flussOnlyPkPlan.contains("unionRead=no"), "the lake should be off: ${flussOnlyPkPlan}") + assertEquals(3, countIn(flussOnlyPkPlan, "pkRanges"), "one range per bucket: ${flussOnlyPkPlan}") + order_qt_fluss_only_pk """ + select count(*) as rows_read, sum(id) as id_sum, max(id) as id_max from big_pk + """ + + sql """switch internal""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy new file mode 100644 index 00000000000000..e7cd43d91f7135 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The rest of what a catalog has to answer for: tables with nothing in them, the +// ways a query over one is supposed to fail, and the ordinary SQL a user reaches +// for once the table reads at all. +// +// The suites next to this one each follow one path in depth. This one is the +// breadth: a fluss table has to behave like a table -- joinable, insertable-from, +// a subquery, one side of a union -- and it has to fail in a way that says what to +// do, on the handful of things it genuinely cannot do. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_misc", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_misc" + String requiredCatalog = "test_fluss_misc_required" + String internalDb = "test_fluss_misc_internal" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + + // --- tables with nothing in them ------------------------------------------ + // A primary-key table that was never written to is not the same shape as an empty + // log table: its read starts from a kv snapshot, and there is none, so the code that + // has to notice is not the code that reads no log records. + order_qt_pk_empty_count """select count(*) from pk_empty""" + order_qt_pk_empty_rows """select id, name from pk_empty""" + def pkEmptyPlan = planOf("""select * from pk_empty""") + // An engine that drops a split-less scan altogether is as correct as one that keeps + // the node, so what is pinned is that no range was planned either way. + def pkEmptyMatcher = (pkEmptyPlan =~ /pkRanges=(\d+)/) + assertTrue(!pkEmptyMatcher.find() || pkEmptyMatcher.group(1) == "0", + "an empty primary-key table planned ranges: ${pkEmptyPlan}") + + // Aggregates over an empty table, where COUNT and the rest disagree by design: + // count is 0 and every other aggregate is NULL, and a scanner that reported an + // empty batch as "no rows read" rather than "zero rows" gets one of them wrong. + order_qt_empty_aggregates """ + select count(*), count(id), sum(id), min(id), max(id), avg(id) from pk_empty + """ + order_qt_log_empty_aggregates """ + select count(*), count(id), sum(id), min(id), max(id) from log_empty + """ + order_qt_empty_join """ + select count(*) from log_empty a join log_basic b on a.id = b.id + """ + + // A lake table tiering has never committed anything for. Under auto there is + // nothing to read the lake half from, so the fluss-only read is the whole answer -- + // which for a table nobody wrote to is no rows. + def lakeEmptyPlan = planOf("""select * from lake_empty""") + assertTrue(lakeEmptyPlan.contains("unionRead=no"), + "there is no lake snapshot to read: ${lakeEmptyPlan}") + assertTrue(lakeEmptyPlan.contains("lakeSplits=0"), "no lake half: ${lakeEmptyPlan}") + order_qt_lake_empty_count """select count(*) from lake_empty""" + + // Under required the same table is an error: falling back is precisely what that + // mode forbids, and "wait for the tiering service" is the thing its user needs to + // be told rather than left to infer from an empty result. + sql """drop catalog if exists ${requiredCatalog}""" + sql """ + create catalog ${requiredCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + test { + sql """select * from ${requiredCatalog}.fluss_test.lake_empty""" + exception "has no readable lake snapshot yet" + } + + // --- the one fluss type Doris has nowhere to put -------------------------- + // TIME is marked unsupported rather than mapped to a string or to elapsed millis: + // either of those hands back a value whose meaning differs from the source, and + // nothing downstream would ever reveal it. So the column is describable and + // unreadable, and the rest of the table is unaffected -- which is what makes the + // marker useful instead of merely honest. + qt_desc_log_time """desc log_time""" + order_qt_time_other_columns """select id, name from log_time""" + order_qt_time_count """select count(*) from log_time""" + test { + sql """select f_time from log_time""" + exception "UNSUPPORTED" + } + test { + sql """select * from log_time""" + exception "UNSUPPORTED" + } + + // --- things that are simply not there ------------------------------------- + test { + sql """select * from no_such_table""" + exception "no_such_table" + } + test { + sql """select no_such_column from log_basic""" + exception "no_such_column" + } + test { + sql """select * from no_such_db.log_basic""" + exception "no_such_db" + } + + // --- a fluss table is a table --------------------------------------------- + // Two fluss tables joined to each other. The scan of each is the same one the other + // suites make; what is new is two of them in one fragment, which is where a scanner + // holding state on something shared between nodes would show up. + order_qt_join_fluss """ + select b.id, b.name, t.f_string + from log_basic b join log_types t on b.id = t.id + """ + order_qt_join_pk """ + select p.id, p.name, l.name as log_name + from pk_basic p join log_basic l on p.id = l.id + """ + // A join over the seam of a table read as lake plus log: rows from both halves have + // to reach the join operator, not just the half that was planned first. + order_qt_join_lake """ + select l.id, l.name, c.name as cold_name + from lake_log l left join lake_cold c on l.id = c.id + """ + order_qt_join_self """ + select a.id, a.name, b.name + from log_part a join log_part b on a.dt = b.dt and a.id < b.id + """ + + // Subqueries, correlated and not. + order_qt_subquery_in """ + select id, name from log_basic where id in (select id from pk_basic where score > 2.0) + """ + order_qt_subquery_exists """ + select id from log_basic b where exists (select 1 from log_types t where t.id = b.id) + """ + order_qt_subquery_scalar """ + select id, name from log_basic where price > (select min(price) from log_basic) + """ + order_qt_cte """ + with hot as (select id, name from lake_log where id >= 5) + select count(*), min(id) from hot + """ + + // Set operations, where a fluss scan is one arm of several. + order_qt_union_all """ + select id, name from log_basic union all select id, name from lake_cold + """ + order_qt_union_distinct """ + select id from log_basic union select id from lake_cold + """ + order_qt_except """ + select id from lake_log except select id from lake_cold + """ + order_qt_intersect """ + select id from lake_log intersect select id from lake_cold + """ + + // Ordinary aggregation and window shapes over a fluss scan. + order_qt_aggregates """ + select count(*), count(distinct name), min(price), max(price), sum(price), avg(id) + from log_basic + """ + order_qt_group_having """ + select dt, count(*) as c from log_part group by dt having count(*) > 1 + """ + order_qt_order_limit """select id, name from log_basic order by price desc limit 2""" + order_qt_limit_offset """select id from log_basic order by id limit 1 offset 1""" + order_qt_star """select * from log_basic""" + order_qt_expression """select id * 2 as doubled, upper(name) from log_basic where id < 3""" + order_qt_case """ + select id, case when price > 20 then 'high' else 'low' end as band from log_basic + """ + + // --- reading a fluss table into an internal one --------------------------- + // The whole point of the connector for most users, and the one path where the + // scanned rows leave the query that scanned them: they are typed, materialized and + // written. A type that reads correctly but describes itself wrongly fails here and + // nowhere else. + sql """switch internal""" + sql """drop database if exists ${internalDb}""" + sql """create database ${internalDb}""" + sql """use ${internalDb}""" + + sql """ + create table copied ( + id int, + name varchar(64), + price decimalv3(10, 2) + ) distributed by hash(id) buckets 1 properties ("replication_num" = "1") + """ + sql """ + insert into copied select id, name, price from ${catalogName}.fluss_test.log_basic + """ + order_qt_copied """select id, name, price from copied""" + + // CTAS, which takes the column types from the connector rather than from a table + // someone wrote by hand -- so this is where the declared Doris type of every mapped + // fluss type has to be a type Doris can actually create a column of. + sql """ + create table ctas_types properties ("replication_num" = "1") as + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from ${catalogName}.fluss_test.log_types + """ + qt_desc_ctas """desc ctas_types""" + order_qt_ctas_count """select count(*), count(f_string), count(f_array) from ctas_types""" + + sql """ + create table ctas_nested properties ("replication_num" = "1") as + select id, f_arr_arr, f_map_row, f_row_deep + from ${catalogName}.fluss_test.log_nested where id = 1 + """ + qt_desc_ctas_nested """desc ctas_nested""" + order_qt_ctas_nested_rows """select id, f_arr_arr, f_map_row, f_row_deep from ctas_nested""" + + // A join between an internal table and a fluss one, which is the shape a user + // actually writes: the two scans are entirely different node types in one plan. + order_qt_join_internal """ + select c.id, c.name, t.f_string + from copied c join ${catalogName}.fluss_test.log_types t on c.id = t.id + """ + + sql """drop database if exists ${internalDb} force""" + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${requiredCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy new file mode 100644 index 00000000000000..57abbac7c7c6c0 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Complex types nested inside complex types, down every path a value can take. +// +// The other type suites cover one column per fluss type, never more than one level +// deep. That is a different question: an element decoder is chosen per level, so a +// reader can be right about MAP and wrong about the MAP> +// beside it, and being right about ARRAY says nothing about ARRAY>. +// Fluss stores these in three unrelated row formats -- arrow for a log table, +// compacted for a primary-key one, parquet through paimon for the lake half -- and +// each has its own nested decoder, so all three are read here. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql (log_nested, +// pk_nested, lake_nested) and init-lake-tail.sql. +suite("test_fluss_nested_types", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_nested_types" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only, and fuzzy sessions + // randomize the variable that chooses between them. + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + + // --- the mapping, before any row is read --------------------------------- + // Recorded unsorted: which level of nesting each constructor sits at is the whole + // subject, and a type printed one level flat is the bug this suite is about. + qt_desc_log_nested """desc log_nested""" + qt_desc_pk_nested """desc pk_nested""" + + // --- a log table's arrow rows -------------------------------------------- + // All three rows whole: the populated one, the one whose collections are present + // but hold NULL elements, and the one whose columns are NULL outright. The middle + // row is the one worth having -- a decoder that confuses "no element" with "a null + // element" gets the other two right and this one wrong. + order_qt_log_rows """ + select id, f_arr_arr, f_arr_map, f_arr_row, f_map_arr, f_map_row, f_row_deep, + f_arr_arr_arr + from log_nested + """ + + // Element by element, so a value that rendered plausibly but decoded into the wrong + // slot has somewhere to fail. Reaching two levels in at once is the point: the + // outer accessor has to hand the inner one a value it can still take apart. + order_qt_log_elements """ + select id, + array_size(f_arr_arr), f_arr_arr[1], array_size(f_arr_arr[1]), f_arr_arr[1][2], + f_arr_map[1]['a'], + struct_element(f_arr_row[1], 'a'), struct_element(f_arr_row[1], 'b'), + f_map_arr['k1'], f_map_arr['k1'][2], + struct_element(f_map_row['k1'], 'a'), struct_element(f_map_row['k1'], 'b') + from log_nested + """ + + // Three levels of the same constructor, and a struct holding one of each. Nothing + // else here asks a decoder to recurse into itself twice. + order_qt_log_deep """ + select id, + f_arr_arr_arr[1][2][1], + struct_element(f_row_deep, 'r_int'), + struct_element(f_row_deep, 'r_arr')[2], + struct_element(f_row_deep, 'r_map')['m'], + struct_element(struct_element(f_row_deep, 'r_row'), 'x'), + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from log_nested + """ + + // A predicate on a nested element, which is evaluated after the value has been + // decoded rather than in the scanner: the row it selects is the check. + order_qt_log_nested_predicate """ + select id from log_nested + where f_arr_arr[1][1] = 1 and struct_element(f_map_row['k1'], 'a') = 9 + """ + + // --- a primary-key table's compacted rows --------------------------------- + // Key 1 was written twice, so what comes back must be the SECOND set of nested + // values. A reader that returned the change log rather than the merged view would + // answer with three rows here, two of them the same key. + order_qt_pk_rows """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from pk_nested + """ + order_qt_pk_elements """ + select id, + array_size(f_arr_arr), f_arr_arr[1][1], + struct_element(f_arr_row[1], 'b'), + f_map_arr['k9'], f_map_arr['k9'][1], + struct_element(f_map_row['k9'], 'a'), + struct_element(f_row_deep, 'r_arr')[1], + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from pk_nested + """ + + // --- the same nesting, through paimon ------------------------------------- + // The connector's fluss->Doris mapping has to equal fluss->paimon->Doris, or one + // table has two schemas depending on which door it is read through. For flat types + // that is checked in test_fluss_lake_only; nesting is where the two mappings could + // most easily part company, since each level is converted by its own rule on both + // sides. + qt_desc_lake_nested """desc lake_nested""" + qt_desc_lake_nested_lake """desc lake_nested\$lake""" + def typesOf = { String table -> + def result = [:] + sql("""desc ${table}""").each { row -> result.put(row[0].toString(), row[1].toString()) } + return result + } + def flussTypes = typesOf("lake_nested") + def lakeTypes = typesOf("lake_nested\$lake") + flussTypes.each { column, type -> + assertEquals(type, lakeTypes.get(column), + "column ${column} is ${type} on the fluss table but ${lakeTypes.get(column)} on its lake") + } + // The lake table carries three system columns fluss adds to every one it creates + // (__bucket, __offset, __timestamp) and nothing else, so the count is the other + // half of the parity: a column present on one side only would satisfy the loop. + assertEquals(flussTypes.size() + 3, lakeTypes.size(), + "the lake table should differ by exactly the three system columns") + + // The tiered row read through paimon's own reader. + order_qt_lake_side """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from lake_nested\$lake + """ + + // And the front door, where the tiered row arrives through paimon and the all-NULL + // one through fluss: two nested decoders feeding one result set. A level dropped by + // either shows up as a value in the wrong shape next to the same column from the + // other half. + def lakePlan = sql("""explain select * from lake_nested""").collect { it[0].toString() }.join("\n") + assertTrue(lakePlan.contains("unionRead=yes"), "not a union read: ${lakePlan}") + order_qt_lake_union """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from lake_nested + """ + order_qt_lake_union_elements """ + select id, f_arr_arr[1][2], struct_element(f_arr_row[1], 'b'), + f_map_arr['k1'][1], struct_element(f_map_row['k1'], 'b'), + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from lake_nested + """ + + // The union read and a reader that never touches the lake must agree row for row. + // Two entirely different nested decoders over one table: the comparison says + // something no pair of recorded blocks does. + String flussOnlyCatalog = "test_fluss_nested_types_off" + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + def unionRows = rowsOf("""select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep + from lake_nested order by id""") + def flussOnlyRows = rowsOf("""select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep + from ${flussOnlyCatalog}.fluss_test.lake_nested order by id""") + assertEquals(flussOnlyRows, unionRows, + "paimon and fluss decoded the same nested rows differently\nfluss-only=${flussOnlyRows}\nunion=${unionRows}") + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy new file mode 100644 index 00000000000000..66b19cb1fe39ac --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Partition columns of every type, and the line between the ones that can be read +// and the ones that cannot. +// +// A fluss partition carries its value nowhere but in its own name, and fluss allows +// only ASCII letters, digits, '_' and '-' there -- so a value holding anything else +// is rewritten on the way in and cannot be recovered. Which types survive that is +// not a matter of taste, and every verdict below was established against this +// cluster: STRING, CHAR, BOOLEAN, the integer family, DATE and (as text) BINARY come +// back as written; FLOAT, DOUBLE, TIME and the timestamps do not. +// +// The other suites partition by STRING only, which is also the one type that cannot +// tell a rendering bug from a working one -- a number, a date or a padded CHAR is +// where the two sides of that rendering could disagree. +// +// Fixtures: part_types, part_ts, lake_part_int, lake_pk_part_int in +// docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_partition_types", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_partition_types" + String varbinaryCatalog = "test_fluss_partition_types_vb" + String unionCatalog = "test_fluss_partition_types_union" + String flussOnlyCatalog = "test_fluss_partition_types_off" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + + // --- every readable partition type, in one table -------------------------- + // Recorded unsorted, because the column ORDER is what fe-core zips the partition + // values against positionally: a listing whose values are right but in another + // order assigns each partition someone else's value, silently. + qt_desc_part_types """desc part_types""" + + // The partition names as this connector renders them. Nothing else pins the + // rendering of a boolean, a date or a byte string into a partition name, and both + // the listing and split planning read this same rendering -- so a change here that + // reached only one of them would prune to nothing while still looking planned. + order_qt_part_names """show partitions from part_types""" + + // The values themselves, next to the row they belong to. The partition columns are + // not read by the scanner at all: FE declares them and BE fills each one in from + // the range it came with, so a value on the wrong split shows up only here. + // + // p_bin records as the hex of "0102" rather than of the bytes 0x01 0x02, and that is + // the truth of a partitioned BINARY rather than a defect: what fluss kept is the + // NAME it gave the partition, which is the hex text, and the value in the row is + // that text. A non-partitioned BINARY column, whose bytes are stored in the row + // itself, reads back as the bytes -- covered in the type suites. + order_qt_part_rows """ + select id, name, p_str, p_char, p_bool, p_tiny, p_small, p_int, p_big, p_date, + p_bin, hex(p_bin) as p_bin_hex + from part_types + """ + + // --- pruning, one partition column at a time ------------------------------ + // Each column takes a different value in each of the two partitions, so a predicate + // on any one of them must leave exactly one. Pruning has to shrink the WORK and not + // only the plan line: a name rendered one way for the listing and another for the + // match reports 1/2 here while scanning nothing at all, which is why the recorded + // rows sit beside the plan assertion rather than instead of it. + def prunesToOne = { String column, String predicate -> + def plan = planOf("""select * from part_types where ${predicate}""") + assertTrue(plan.contains("partition=1/2"), + "pruning on ${column} did not reach the connector: ${plan}") + assertTrue(countIn(plan, "logRanges") >= 1, + "pruning on ${column} left no range to read: ${plan}") + } + prunesToOne("p_str", "p_str = 'cn'") + prunesToOne("p_char", "p_char = 'c1'") + prunesToOne("p_bool", "p_bool = true") + prunesToOne("p_tiny", "p_tiny = 1") + prunesToOne("p_small", "p_small = 10") + prunesToOne("p_int", "p_int = 100") + prunesToOne("p_big", "p_big = 1000") + prunesToOne("p_date", "p_date = '2026-01-01'") + + order_qt_pruned_str """select id, name from part_types where p_str = 'cn'""" + order_qt_pruned_int """select id, name from part_types where p_int = 200""" + order_qt_pruned_date """select id, name from part_types where p_date = '2026-01-02'""" + order_qt_pruned_bool """select id, name from part_types where p_bool = false""" + order_qt_pruned_tiny """select id, name from part_types where p_tiny = 2""" + + // Several partition columns at once, which is also the ordinary case for this + // table: they have to agree rather than each prune on its own. + order_qt_pruned_multi """ + select id, name from part_types where p_str = 'cn' and p_int = 100 and p_bool = true + """ + // A combination no partition holds. Pruning to nothing is a correct answer and not + // an error, and it must not be reached by scanning everything and filtering after. + def emptyPlan = planOf("""select * from part_types where p_str = 'cn' and p_int = 200""") + assertTrue(emptyPlan.contains("partition=0/2"), + "an impossible combination should prune to nothing: ${emptyPlan}") + order_qt_pruned_impossible """ + select count(*) from part_types where p_str = 'cn' and p_int = 200 + """ + order_qt_pruned_absent """select count(*) from part_types where p_int = 999""" + + // --- the type whose value fluss cannot store ------------------------------ + // Fluss creates such a table happily and names the partition of + // 2026-01-01 01:02:03.0 as 2026-01-01-01-02-03_0 -- every character it may not + // hold rewritten, many-to-one. Left alone this reaches fe-core's partition parser, + // which fails with the mangled name and nothing else: no column, no type, no fluss. + // The refusal therefore belongs to the connector, and has to name all three. + test { + sql """select * from part_ts""" + exception "its partition column 'p_ts' has fluss type TIMESTAMP(3)" + } + test { + sql """show partitions from part_ts""" + exception "cannot be read back" + } + // DESC still works: the table is describable, it is only unreadable, and the column + // its user has to change is the one this shows. + qt_desc_part_ts """desc part_ts""" + + // --- the type whose verdict the catalog decides --------------------------- + // Fluss names a BINARY partition with the hex text of the bytes. Read as text that + // is exactly what was written; asked for as a VARBINARY it is not a literal of + // anything, so the same table is readable through one catalog and not the other. + sql """drop catalog if exists ${varbinaryCatalog}""" + sql """ + create catalog ${varbinaryCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "enable.mapping.varbinary" = "true" + ); + """ + test { + sql """select id from ${varbinaryCatalog}.fluss_test.part_types""" + exception "enable.mapping.varbinary" + } + + // --- a non-STRING partition under a union read ---------------------------- + // Concatenating a lake with the log written after it needs no partition value + // matched across the halves: each half prunes on its own. So an INT partition works + // here, and this is what says the rule that stops the primary-key merge below was + // not quietly applied to everything partitioned. + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def intPartPlan = planOf("""select * from lake_part_int""") + assertTrue(intPartPlan.contains("unionRead=yes"), + "an INT-partitioned log table should still be read as lake plus log: ${intPartPlan}") + assertTrue(countIn(intPartPlan, "lakeSplits") >= 1, "no lake splits: ${intPartPlan}") + assertEquals(1, countIn(intPartPlan, "logRanges"), + "one partition has a tail and the other does not: ${intPartPlan}") + order_qt_lake_part_int """select id, name, p_int from lake_part_int""" + + // Both halves must land on the same partition: one of these has a tail, the other + // is served entirely from the lake. + def tailPlan = planOf("""select * from lake_part_int where p_int = 1""") + assertTrue(tailPlan.contains("partition=1/2"), "pruning did not reach the connector: ${tailPlan}") + assertEquals(1, countIn(tailPlan, "logRanges"), "the partition with a tail lost it: ${tailPlan}") + order_qt_lake_part_int_tail """select id, name from lake_part_int where p_int = 1""" + + def tieredPlan = planOf("""select * from lake_part_int where p_int = 2""") + assertEquals(0, countIn(tieredPlan, "logRanges"), + "a fully tiered partition still has log ranges: ${tieredPlan}") + order_qt_lake_part_int_tiered """select id, name from lake_part_int where p_int = 2""" + + def unionIntRows = rowsOf("""select id, name, p_int from lake_part_int order by id""") + def flussOnlyIntRows = rowsOf( + """select id, name, p_int from ${flussOnlyCatalog}.fluss_test.lake_part_int order by id""") + assertEquals(flussOnlyIntRows, unionIntRows, + "lake+log and fluss-only disagree\nfluss-only=${flussOnlyIntRows}\nunion=${unionIntRows}") + + // --- a non-STRING partition under a merge BY KEY -------------------------- + // Merging the halves of a primary-key table means deciding which fluss partition a + // paimon split belongs to, by comparing the text each side renders that value as -- + // and only STRING is guaranteed to render alike on both. So this table is not + // merged. Under `required` that is an error naming the reason; the fluss-only read + // it would otherwise fall back to returns every row anyway, so nothing is lost but + // the lake's speed. + test { + sql """select * from lake_pk_part_int""" + exception "cannot be read as its lake plus its change log" + } + + // Under `auto` it falls back instead, and says so in the plan. Without the anchor + // the fallback and a working merge look identical from the rows alone -- which is + // exactly the point of the fallback. + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + def degradedPlan = planOf("""select * from lake_pk_part_int""") + assertTrue(degradedPlan.contains("unionRead=no"), + "the halves should not have been merged: ${degradedPlan}") + assertTrue(degradedPlan.contains("degraded=partition-type"), + "the plan should say why it fell back: ${degradedPlan}") + assertEquals(0, countIn(degradedPlan, "lakeSplits"), "no lake half is read: ${degradedPlan}") + assertTrue(countIn(degradedPlan, "pkRanges") >= 1, "the whole table comes from fluss: ${degradedPlan}") + + // And the rows are still all of them -- a primary-key table's fluss-only read is the + // whole table, not the part the lake has not taken yet. + order_qt_lake_pk_part_int """select id, name, p_int from lake_pk_part_int""" + order_qt_lake_pk_part_int_pruned """select id, name from lake_pk_part_int where p_int = 1""" + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${varbinaryCatalog}""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy new file mode 100644 index 00000000000000..15d402f4eba328 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Predicates that are more than one comparison. +// +// The other suites filter with a single predicate, which is the shape that cannot +// go wrong in an interesting way. Combinations can: a disjunction over a partition +// column has to keep every partition it names rather than the last one; a +// conjunction of a partition predicate and a data predicate must prune on the first +// without dropping rows the second would have kept; a NOT around a partition +// predicate must not prune at all. And for a table read as lake plus log, a +// predicate that reached only one half returns an answer that is short by exactly +// the other half's matching rows -- a plausible number, always. +// +// This suite pins WHAT the answers are; where a predicate is evaluated is not +// asserted, because it is the optimizer's choice and pinning it would fail on the +// day the optimizer gets better rather than worse. The union-read blocks compare +// the two read paths instead, which is the property that has to hold however the +// predicate was routed. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_predicates", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_predicates" + String flussOnlyCatalog = "test_fluss_predicates_off" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + // The timestamp columns below are compared against literals, and TIMESTAMP_LTZ + // renders through the session time zone. + sql """set time_zone = 'Asia/Shanghai'""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def compareModes = { String query -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + } + + // --- conjunction, disjunction, negation on a log table -------------------- + order_qt_and """select id, name from log_basic where id > 1 and price < 25.00""" + order_qt_or """select id, name from log_basic where id = 1 or price > 25.00""" + // The parenthesised mixture: a reader that flattened it into a chain of ANDs or + // ORs answers with everything or nothing, and both look like working code. + order_qt_and_or """ + select id, name from log_basic + where (id = 1 and price < 15.00) or (id = 3 and price > 25.00) + """ + order_qt_or_and """ + select id, name from log_basic + where (id = 1 or id = 3) and price > 25.00 + """ + order_qt_not """select id, name from log_basic where not (id = 1 or price > 25.00)""" + order_qt_not_between """select id from log_basic where id not between 2 and 3""" + order_qt_in """select id, name from log_basic where id in (1, 3) and name <> 'carol'""" + order_qt_not_in """select id from log_basic where id not in (1, 2)""" + order_qt_like """select id from log_basic where name like '%o%' and name not like 'c%'""" + // A predicate that selects nothing, and one that selects everything: the two ends + // of the range, where an inverted condition is easiest to mistake for a working one. + order_qt_always_false """select count(*) from log_basic where id > 0 and id < 0""" + order_qt_always_true """select count(*) from log_basic where id > 0 or name is not null""" + + // --- NULLs, which no comparison is true of -------------------------------- + // The all-NULL row is row 3 of log_types. `f_int <> 3` must not return it, `is null` + // must, and `not (f_int = 3)` must not either -- three-valued logic is exactly what + // a filter pushed down and re-implemented gets wrong. + order_qt_is_null """select id from log_types where f_int is null""" + order_qt_is_not_null """select id from log_types where f_int is not null""" + order_qt_ne_excludes_null """select id from log_types where f_int <> 3""" + order_qt_not_eq_excludes_null """select id from log_types where not (f_int = 3)""" + order_qt_null_or """select id from log_types where f_int is null or f_int > 0""" + order_qt_null_and """select id from log_types where f_string is null and f_boolean is null""" + + // Across types, in one condition. A predicate list that stopped at the first column + // it could not push would return the rows the rest were meant to exclude. + order_qt_mixed_types """ + select id from log_types + where f_boolean = true and f_tinyint > 0 and f_decimal > 0 and f_char = 'char1' + and f_date = '2026-01-01' and f_string like 'string%' + """ + order_qt_mixed_types_or """ + select id from log_types + where f_bigint = 4 or f_double < 0 or (f_float is null and f_smallint is null) + """ + // Reaching into a complex value from a predicate: the element has to be decoded + // before it can be compared, so this cannot have been answered by the scanner. + order_qt_nested_elements """ + select id from log_types + where f_array[1] = 1 and f_map['k1'] = 1 and struct_element(f_row, 'r_int') = 1 + """ + + // --- a partition column in a compound predicate --------------------------- + // Pruning takes the partitions the predicate leaves possible, so a disjunction has + // to keep both of the partitions it names and a negation has to keep every one it + // does not exclude. The plan line is asserted alongside the rows because a + // conservative pruner returning too many partitions still gives the right rows. + def partOrPlan = planOf("""select * from log_part where dt = '20260101' or dt = '20260103'""") + assertTrue(partOrPlan.contains("partition=2/3"), + "a disjunction over partitions should keep both: ${partOrPlan}") + order_qt_part_or """select id, name, dt from log_part where dt = '20260101' or dt = '20260103'""" + + def partInPlan = planOf("""select * from log_part where dt in ('20260101', '20260102')""") + assertTrue(partInPlan.contains("partition=2/3"), "IN over partitions: ${partInPlan}") + order_qt_part_in """select id, dt from log_part where dt in ('20260101', '20260102') and id > 1""" + + def partNotPlan = planOf("""select * from log_part where dt <> '20260101'""") + assertTrue(partNotPlan.contains("partition=2/3"), "a negated partition predicate: ${partNotPlan}") + order_qt_part_ne """select id, dt from log_part where dt <> '20260101'""" + + // A partition predicate and a data predicate together: the first prunes, the second + // must still be applied to what is left rather than assumed satisfied. + def partAndDataPlan = planOf("""select * from log_part where dt = '20260101' and id = 2""") + assertTrue(partAndDataPlan.contains("partition=1/3"), "pruning still applies: ${partAndDataPlan}") + order_qt_part_and_data """select id, name from log_part where dt = '20260101' and id = 2""" + + // A disjunction that mixes the two cannot prune at all: the row it selects lives in + // a partition the partition predicate excludes. + def partOrDataPlan = planOf("""select * from log_part where dt = '20260101' or id = 4""") + assertTrue(partOrDataPlan.contains("partition=3/3"), + "a disjunction with a data predicate must not prune: ${partOrDataPlan}") + order_qt_part_or_data """select id, name, dt from log_part where dt = '20260101' or id = 4""" + + order_qt_part_absent """select count(*) from log_part where dt = '20261231'""" + + // --- a primary-key table, where a predicate is applied after the merge ---- + // The change log holds superseded and deleted rows; a predicate evaluated against + // it rather than against the merged view brings them back. 'k2' is the value key 2 + // used to have, and key 3 was deleted. + order_qt_pk_and """select id, name from pk_basic where id >= 2 and score > 3.0""" + order_qt_pk_or """select id, name from pk_basic where name = 'k1' or score > 20.0""" + order_qt_pk_stale_value """select count(*) from pk_basic where name = 'k2' or id = 3""" + order_qt_pk_not """select id, name from pk_basic where not (id = 1) and name is not null""" + order_qt_pk_in """select id, score from pk_basic where id in (1, 2, 3) and score is not null""" + + // --- a table read as lake plus log ---------------------------------------- + // Whichever half the optimizer pushed the predicate to, both halves have to end up + // filtered by it. The comparison is the check: a predicate that reached only the + // lake returns the log's matching rows unfiltered, and one that reached only the log + // returns the lake's -- either way a plausible count. + order_qt_union_and """select id, name from lake_log where id > 2 and price < 6.00""" + compareModes("""select id, name from lake_log where id > 2 and price < 6.00 order by id""") + + order_qt_union_or """select id, name from lake_log where id = 1 or id = 6""" + compareModes("""select id, name from lake_log where id = 1 or id = 6 order by id""") + + order_qt_union_not """select id from lake_log where not (id between 2 and 5)""" + compareModes("""select id from lake_log where not (id between 2 and 5) order by id""") + + order_qt_union_like """select id, name from lake_log where name like 'hot%' or name like '%1'""" + compareModes("""select id, name from lake_log where name like 'hot%' or name like '%1' order by id""") + + // A predicate that only the lake half can satisfy, and one that only the log half + // can: each is the case where returning the other half unfiltered is invisible in + // the row count of the first. + order_qt_union_lake_only """select id, name from lake_log where id <= 4 and price < 5.00""" + order_qt_union_log_only """select id, name from lake_log where id >= 5""" + order_qt_union_neither """select count(*) from lake_log where id > 100""" + + // The same over a partitioned lake table, where pruning and the seam interact: one + // partition has a tail and the other does not. + order_qt_union_part_or """ + select id, name, dt from lake_part where dt = '20260101' or dt = '20260102' + """ + compareModes(""" + select id, name, dt from lake_part where dt = '20260101' or dt = '20260102' order by id + """) + order_qt_union_part_and """select id from lake_part where dt = '20260101' and id > 2""" + compareModes("""select id from lake_part where dt = '20260101' and id > 2 order by id""") + + // And over a primary-key lake table, where the predicate is applied to the merged + // view of both halves: key 1 was deleted by the tail and key 3 updated by it. + order_qt_union_pk_and """select id, name from lake_pk where id >= 2 and name like '%hot'""" + compareModes("""select id, name from lake_pk where id >= 2 and name like '%hot' order by id""") + order_qt_union_pk_or """select id, name from lake_pk where id = 1 or id = 4""" + compareModes("""select id, name from lake_pk where id = 1 or id = 4 order by id""") + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +}