From db6dd66794a99a6fb084fd41b0f30039ce2ccb3b Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 5 Oct 2026 13:12:01 +0800 Subject: [PATCH 01/15] [jindo] Route reads to io-cache targets by file type When the catalog options carry io-cache endpoints instead of a JindoCache RPC address, JindoFileIO now reads immutable files through a cache target chosen by file type and keeps every other request on the OSS endpoint. Before, it logged "fallback to no-cache" and used OSS for everything. - io-cache.endpoint names one target. io-cache.targets with io-cache.target..endpoint (and optional .region and .path-style-access) names several, and io-cache.routes maps file types to them. io-cache.whitelist and io-cache.policy keep their meaning, and io-cache.origin.endpoint names the OSS endpoint when fs.oss.endpoint points at a cache for older clients. - Only read and meta requests use a target; exists counts as meta. Writes, including the existence check before a two-phase write, listings, deletes, renames, copies and non-oss paths stay on OSS, and a client-side dlf.oss-endpoint turns routing off. - Files rewritten in place (FileType.isMutable) and metadata named by a sequence id never use a target. - Data files are recognized by name: a data or changelog file prefix, including a custom one, followed by {uuid}-{count}.{extension}. Manifests, indexes and statistics share the {uuid}-{count} shape but have no extension, so a custom prefix such as manifest- cannot mix them up; the manifest sidecar is excluded explicitly. Other names keep their FileType, and names Paimon does not write stay on OSS. - Errors from a target are thrown as they are, without retrying on OSS. A target that fails to initialize throws the same checked IOException as the OSS file system does, so retries and fallbacks still apply. - The JindoCache RPC mode is unchanged. The routing cases in src/test/resources/io-cache are shared with pypaimon and paimon-rust. --- .../paimon/jindo/HadoopCompliantFileIO.java | 70 ++- .../apache/paimon/jindo/IoCacheRouting.java | 338 +++++++++++++ .../org/apache/paimon/jindo/JindoFileIO.java | 111 ++++- .../paimon/jindo/IoCacheRoutingTest.java | 286 +++++++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 445 ++++++++++++++++++ .../paimon/jindo/TestJindoCacheEnable.java | 16 + .../resources/io-cache/routable-types.json | 69 +++ .../src/test/resources/io-cache/routing.json | 90 ++++ pom.xml | 2 + 9 files changed, 1402 insertions(+), 25 deletions(-) create mode 100644 paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java create mode 100644 paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java create mode 100644 paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java create mode 100644 paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json create mode 100644 paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 64fc3e07c708..8d535ebc267f 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.options.Options; import org.apache.paimon.utils.Pair; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -37,6 +38,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -58,6 +61,7 @@ public abstract class HadoopCompliantFileIO implements FileIO { private static final Logger LOG = LoggerFactory.getLogger(HadoopCompliantFileIO.class); private static final long serialVersionUID = 1L; + private static final String JINDO_CACHE_RPC_ADDRESS = "fs.jindocache.namespace.rpc.address"; /// Detailed cache strategies are retrieved from REST server. private static final String META_CACHE_ENABLED_TAG = "meta"; @@ -72,10 +76,16 @@ public abstract class HadoopCompliantFileIO implements FileIO { protected transient volatile Map> fsMap; protected transient volatile Map> jindoCacheFsMap; + // Non-null when endpoint routing is configured instead of JindoCache RPC. + @Nullable IoCacheRouting cacheRouting; + // Only enable cache for path which is generated with uuid private List cacheWhitelistPaths = new ArrayList<>(); boolean shouldCache(Path path) { + if (cacheRouting != null) { + return cacheRouting.targetOf(path) != null; + } if (cacheWhitelistPaths.isEmpty()) { return true; } @@ -90,28 +100,40 @@ boolean shouldCache(Path path) { @Override public void configure(CatalogContext context) { - // Process file io cache configuration - if (!context.options().get(IO_CACHE_ENABLED) - || context.options().get(IO_CACHE_POLICY) == null - || context.options().get(IO_CACHE_POLICY).contains(DISABLE_CACHE_TAG)) { + Options options = context.options(); + // Configure cache for the selected backend. + if (!options.get(IO_CACHE_ENABLED) || options.get(IO_CACHE_POLICY) == null) { LOG.debug( "Cache is disabled with io-cache.enabled={}, io-cache.policy={}", - context.options().get(IO_CACHE_ENABLED), - context.options().get(IO_CACHE_POLICY)); + options.get(IO_CACHE_ENABLED), + options.get(IO_CACHE_POLICY)); return; } - // Enable file io cache - if (context.options().get("fs.jindocache.namespace.rpc.address") == null) { + if (options.get(JINDO_CACHE_RPC_ADDRESS) == null) { + cacheRouting = IoCacheRouting.create(options); + if (cacheRouting == null) { + LOG.debug( + "FileIO cache endpoint routing is not configured for io-cache.policy={}", + options.get(IO_CACHE_POLICY)); + return; + } + // Writes always go to the OSS endpoint, so writeCacheEnabled stays false. + metaCacheEnabled = cacheRouting.metaCacheEnabled(); + readCacheEnabled = cacheRouting.readCacheEnabled(); LOG.info( - "FileIO cache is enabled but JindoCache RPC address is not set, fallback to no-cache"); + "Cache endpoints enabled: meta cache enabled {}, read cache enabled {}, {}", + metaCacheEnabled, + readCacheEnabled, + cacheRouting); } else { - metaCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(META_CACHE_ENABLED_TAG); - readCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(READ_CACHE_ENABLED_TAG); - writeCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(WRITE_CACHE_ENABLED_TAG); - String whitelist = context.options().get(IO_CACHE_WHITELIST_PATH); + // Keep legacy JindoCache policy matching unchanged; endpoint routing uses exact tokens. + if (options.get(IO_CACHE_POLICY).contains(DISABLE_CACHE_TAG)) { + return; + } + metaCacheEnabled = options.get(IO_CACHE_POLICY).contains(META_CACHE_ENABLED_TAG); + readCacheEnabled = options.get(IO_CACHE_POLICY).contains(READ_CACHE_ENABLED_TAG); + writeCacheEnabled = options.get(IO_CACHE_POLICY).contains(WRITE_CACHE_ENABLED_TAG); + String whitelist = options.get(IO_CACHE_WHITELIST_PATH); if (!whitelist.equals("*")) { cacheWhitelistPaths = Lists.newArrayList(whitelist.split(",")); } @@ -226,6 +248,22 @@ public boolean rename(Path src, Path dst) throws IOException { return getFileSystem(hadoopSrc, false).rename(hadoopSrc, hadoopDst); } + @Override + public void copyFile(Path sourcePath, Path targetPath, boolean overwrite) throws IOException { + if (cacheRouting == null) { + FileIO.super.copyFile(sourcePath, targetPath, overwrite); + return; + } + // copies read the source from the OSS endpoint, not from a cache endpoint + org.apache.hadoop.fs.Path hadoopSrc = path(sourcePath); + org.apache.hadoop.fs.Path hadoopDst = path(targetPath); + try (FSDataInputStream in = getFileSystem(hadoopSrc, false).open(hadoopSrc); + FSDataOutputStream out = + getFileSystem(hadoopDst, false).create(hadoopDst, overwrite)) { + IOUtils.copyBytes(in, out, 4096); + } + } + protected org.apache.hadoop.fs.Path path(Path path) { URI uri = path.toUri(); if ("oss".equals(uri.getScheme()) && uri.getUserInfo() != null) { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java new file mode 100644 index 000000000000..e8caf99f4059 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.FileType; +import org.apache.paimon.utils.StringUtils; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.apache.paimon.CoreOptions.CHANGELOG_FILE_PREFIX; +import static org.apache.paimon.CoreOptions.DATA_FILE_PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.DLF_OSS_ENDPOINT; +import static org.apache.paimon.rest.RESTCatalogOptions.IO_CACHE_ENABLED; +import static org.apache.paimon.rest.RESTCatalogOptions.IO_CACHE_POLICY; + +/** + * Selects cache targets for immutable Paimon files using {@code io-cache.targets}, {@code + * io-cache.routes} and {@code io-cache.whitelist}. The endpoint mode also carries the normalized + * {@code io-cache.policy} read and metadata switches. + */ +final class IoCacheRouting implements Serializable { + + private static final long serialVersionUID = 1L; + + private static final String META_CACHE_ENABLED_TAG = "meta"; + private static final String READ_CACHE_ENABLED_TAG = "read"; + private static final String DISABLE_CACHE_TAG = "none"; + + private static final Pattern TARGET_NAME = Pattern.compile("[a-z][a-z0-9-]*"); + + // Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the + // prefix. + private static final Pattern DATA_FILE_SUFFIX = + Pattern.compile( + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\\..+"); + + private static final String MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar"; + + @Nullable private final String ossEndpoint; + private final List targets; + private final Map targetByType; + private final List dataFilePrefixes; + private final boolean metaCacheEnabled; + private final boolean readCacheEnabled; + + private IoCacheRouting( + @Nullable String ossEndpoint, + List targets, + Map targetByType, + List dataFilePrefixes, + boolean metaCacheEnabled, + boolean readCacheEnabled) { + this.ossEndpoint = ossEndpoint; + this.targets = targets; + this.targetByType = targetByType; + this.dataFilePrefixes = dataFilePrefixes; + this.metaCacheEnabled = metaCacheEnabled; + this.readCacheEnabled = readCacheEnabled; + } + + /** Creates endpoint routing from the options, or null when routing is not enabled. */ + @Nullable + static IoCacheRouting create(Options options) { + if (!options.get(IO_CACHE_ENABLED)) { + return null; + } + Set policy = parsePolicy(options.get(IO_CACHE_POLICY)); + boolean metaCache = policy.contains(META_CACHE_ENABLED_TAG); + boolean readCache = policy.contains(READ_CACHE_ENABLED_TAG); + if (policy.contains(DISABLE_CACHE_TAG) || !(metaCache || readCache)) { + return null; + } + // a client-side dlf.oss-endpoint replaces every endpoint the token vends + if (!StringUtils.isNullOrWhitespaceOnly(options.get(DLF_OSS_ENDPOINT.key()))) { + return null; + } + Map urls = declaredTargets(options); + if (urls.isEmpty()) { + return null; + } + Map byName = new LinkedHashMap<>(); + urls.forEach( + (name, url) -> { + if (!StringUtils.isNullOrWhitespaceOnly(url)) { + byName.put(name, target(options, name, url.trim())); + } + }); + + String whitelist = options.get("io-cache.whitelist"); + Set types = + whitelist == null ? EnumSet.allOf(FileType.class) : fileTypes(whitelist); + String routes = options.get("io-cache.routes"); + Map targetByType = new EnumMap<>(FileType.class); + for (FileType type : types) { + CacheTarget target = + routes == null + ? byName.values().stream().findFirst().orElse(null) + : byName.get(routedName(routes, type)); + if (target != null) { + targetByType.put(type, target); + } + } + + String origin = options.get("io-cache.origin.endpoint"); + return new IoCacheRouting( + StringUtils.isNullOrWhitespaceOnly(origin) + ? options.get("fs.oss.endpoint") + : origin, + new ArrayList<>(byName.values()), + targetByType, + dataFilePrefixes(options), + metaCache, + readCache); + } + + private static Set parsePolicy(@Nullable String value) { + return value == null + ? Collections.emptySet() + : Arrays.stream(value.split(",")) + .map(token -> token.trim().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + } + + boolean metaCacheEnabled() { + return metaCacheEnabled; + } + + boolean readCacheEnabled() { + return readCacheEnabled; + } + + /** The OSS endpoint used by requests that do not use a cache target. */ + @Nullable + String ossEndpoint() { + return ossEndpoint; + } + + Collection targets() { + return targets; + } + + /** The cache target selected for a file, or null when it does not use a cache target. */ + @Nullable + CacheTarget targetOf(Path path) { + // Only OSS paths can use cache targets. + if (!"oss".equals(path.toUri().getScheme())) { + return null; + } + FileType type = cacheableType(path, dataFilePrefixes); + return type == null ? null : targetByType.get(type); + } + + /** The type of a file a cache may serve, or null when it must be read from OSS. */ + @Nullable + static FileType cacheableType(Path path, List dataFilePrefixes) { + String name = path.getName(); + if (isTemporary(name) || FileType.isMutable(path)) { + return null; + } + if (isDataFileName(name, dataFilePrefixes)) { + return name.endsWith(".index") ? FileType.FILE_INDEX : FileType.DATA; + } + if (isSequential(path)) { + return null; + } + FileType type = FileType.classify(path); + return type == FileType.DATA ? null : type; + } + + // Manifests, indexes and statistics share the uuid-count shape but have no extension. + private static boolean isDataFileName(String name, List dataFilePrefixes) { + if (name.endsWith(MANIFEST_SIDECAR_SUFFIX)) { + return false; + } + for (String prefix : dataFilePrefixes) { + if (name.startsWith(prefix) + && DATA_FILE_SUFFIX.matcher(name.substring(prefix.length())).matches()) { + return true; + } + } + return false; + } + + private static boolean isTemporary(String name) { + return name.endsWith(".tmp") || name.contains(".tmp-") || name.contains(".tmp."); + } + + // Sequential metadata can be read before it exists; a cached NotFound would hide new commits. + private static boolean isSequential(Path path) { + String name = path.getName(); + Path parent = path.getParent(); + return name.startsWith("snapshot-") + || name.startsWith("schema-") + || (name.startsWith("changelog-") + && parent != null + && "changelog".equals(parent.getName())); + } + + static List dataFilePrefixes(Options options) { + List prefixes = new ArrayList<>(); + prefixes.add(DATA_FILE_PREFIX.defaultValue()); + prefixes.add(CHANGELOG_FILE_PREFIX.defaultValue()); + for (String key : new String[] {DATA_FILE_PREFIX.key(), CHANGELOG_FILE_PREFIX.key()}) { + String prefix = options.get(key); + if (!StringUtils.isNullOrWhitespaceOnly(prefix) && !prefixes.contains(prefix)) { + prefixes.add(prefix); + } + } + return prefixes; + } + + // io-cache.targets with io-cache.target..endpoint, else io-cache.endpoint as "default" + private static Map declaredTargets(Options options) { + Map urls = new LinkedHashMap<>(); + String names = options.get("io-cache.targets"); + if (names == null) { + String url = options.get("io-cache.endpoint"); + if (url != null) { + urls.put("default", url); + } + return urls; + } + for (String name : names.toLowerCase(Locale.ROOT).split(",")) { + name = name.trim(); + if (TARGET_NAME.matcher(name).matches() && !urls.containsKey(name)) { + urls.put(name, options.get("io-cache.target." + name + ".endpoint")); + } + } + return urls; + } + + private static CacheTarget target(Options options, String name, String url) { + String prefix = "io-cache.target." + name + "."; + String region = options.get(prefix + "region"); + String pathStyle = options.get(prefix + "path-style-access"); + return new CacheTarget( + name, + url.contains("://") ? url : "https://" + url, + pathStyle != null && "true".equalsIgnoreCase(pathStyle.trim()), + StringUtils.isNullOrWhitespaceOnly(region) ? null : region); + } + + // io-cache.routes is "types=name;...": the first matching rule selects the target. + @Nullable + static String routedName(String routes, FileType type) { + for (String rule : routes.toLowerCase(Locale.ROOT).split(";")) { + int eq = rule.indexOf('='); + String name = eq > 0 ? rule.substring(eq + 1).trim() : ""; + if (!name.isEmpty() && fileTypes(rule.substring(0, eq)).contains(type)) { + return name; + } + } + return null; + } + + private static Set fileTypes(String value) { + return FileType.parseWhitelist(value.toLowerCase(Locale.ROOT)); + } + + @Override + public String toString() { + return "IoCacheRouting{" + targetByType + ", oss=" + ossEndpoint + "}"; + } + + /** A named cache target with its endpoint and addressing settings. */ + static final class CacheTarget implements Serializable { + + private static final long serialVersionUID = 1L; + + final String name; + final String url; + final boolean pathStyleAccess; + @Nullable final String region; + + CacheTarget(String name, String url, boolean pathStyleAccess, @Nullable String region) { + this.name = name; + this.url = url; + this.pathStyleAccess = pathStyleAccess; + this.region = region; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CacheTarget)) { + return false; + } + CacheTarget that = (CacheTarget) o; + return name.equals(that.name) + && url.equals(that.url) + && pathStyleAccess == that.pathStyleAccess + && Objects.equals(region, that.region); + } + + @Override + public int hashCode() { + return Objects.hash(name, url, pathStyleAccess, region); + } + + @Override + public String toString() { + return name + "=" + url; + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 71468672af9f..f8ab0e55e66b 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.HadoopOptionsProvider; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.plugin.PluginLoader; import org.apache.paimon.utils.IOUtils; @@ -40,6 +41,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -72,6 +75,13 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; private static final String OSS_SHOW_DIR_TIMESTAMP = "fs.oss.show-dir-timestamp"; + private static final String OSS_ENDPOINT = "fs.oss.endpoint"; + private static final String OSS_HTTPS_ENABLE = "fs.oss.https.enable"; + private static final String OSS_SECOND_LEVEL_DOMAIN_ENABLE = + "fs.oss.second.level.domain.enable"; + private static final String OSS_REGION = "fs.oss.region"; + // JindoSDK options of a DLF cache cluster, which the OSS endpoint must not use + private static final String OSS_DLF_CACHE_PREFIX = "fs.oss.dlf-cache."; private static final Map CASE_SENSITIVE_KEYS = new HashMap() { @@ -91,6 +101,8 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private Options hadoopOptions; private Options hadoopOptionsWithCache; + private Map cacheTargetOptions; + transient volatile Map> cacheTargetFsMap; private boolean allowCache = true; private transient BlobPresigner blobPresigner; @@ -163,6 +175,38 @@ public void configure(CatalogContext context) { hadoopOptionsWithCache.set("fs.oss.read.profile.columnar.use-pread", "false"); hadoopOptionsWithCache.set( "fs.jindocache.read.profile.columnar.readahead.pread.enable", "false"); + + if (cacheRouting != null) { + cacheTargetOptions = new HashMap<>(); + for (CacheTarget target : cacheRouting.targets()) { + Options options = withEndpoint(hadoopOptions, target.url); + options.set(OSS_SECOND_LEVEL_DOMAIN_ENABLE, String.valueOf(target.pathStyleAccess)); + if (target.region != null) { + options.set(OSS_REGION, target.region); + } + cacheTargetOptions.put(target.name, options); + } + // fs.oss.endpoint may name a cache for older clients, so set the OSS endpoint again + hadoopOptions = withEndpoint(hadoopOptions, cacheRouting.ossEndpoint()); + hadoopOptions.keySet().removeIf(key -> key.startsWith(OSS_DLF_CACHE_PREFIX)); + } + } + + /** Copies the options with an endpoint: its host[:port], and https from its scheme if any. */ + static Options withEndpoint(Options base, @Nullable String endpoint) { + Options options = new Options(base.toMap()); + if (endpoint == null) { + return options; + } + int schemeEnd = endpoint.indexOf("://"); + if (schemeEnd >= 0) { + String scheme = endpoint.substring(0, schemeEnd); + options.set(OSS_HTTPS_ENABLE, String.valueOf("https".equalsIgnoreCase(scheme))); + endpoint = endpoint.substring(schemeEnd + 3); + } + int pathStart = endpoint.indexOf('/'); + options.set(OSS_ENDPOINT, pathStart >= 0 ? endpoint.substring(0, pathStart) : endpoint); + return options; } /** @@ -182,20 +226,27 @@ public Options hadoopOptions(Path path, String opType) { } else if (opType.equalsIgnoreCase("meta")) { shouldCache = metaCacheEnabled && shouldCache(path); } - if (shouldCache) { - return hadoopOptionsWithCache; - } else { + if (!shouldCache) { return hadoopOptions; + } else if (cacheRouting != null) { + return cacheTargetOptions.get(cacheRouting.targetOf(path).name); + } else { + return hadoopOptionsWithCache; } } @Override public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite) throws IOException { - if (!overwrite && this.exists(path)) { + org.apache.hadoop.fs.Path hadoopPath = path(path); + // The check is part of the write, so a cache endpoint never sees the file before it exists. + boolean exists = + cacheRouting == null + ? this.exists(path) + : getFileSystem(hadoopPath, false).exists(hadoopPath); + if (!overwrite && exists) { throw new IOException("File " + path + " already exists."); } - org.apache.hadoop.fs.Path hadoopPath = path(path); Pair pair = getFileSystemPair(hadoopPath, false); JindoHadoopSystem fs = pair.getKey(); JindoMpuStore mpuStore = fs.getMpuStore(hadoopPath); @@ -203,7 +254,8 @@ public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite LOG.debug( "Jindo multipart upload is unavailable for {}, falling back to rename commit.", path); - return super.newTwoPhaseOutputStream(path, overwrite); + // With cache routing the check above already ran on OSS; do not repeat it on a cache. + return super.newTwoPhaseOutputStream(path, overwrite || cacheRouting != null); } return new JindoTwoPhaseOutputStream( new JindoMultiPartUpload(mpuStore, fs.getWorkingDirectory()), hadoopPath, path); @@ -256,9 +308,44 @@ private static synchronized PluginLoader getLoader() { @Override protected Pair createFileSystem( org.apache.hadoop.fs.Path path, boolean enableCache) { + return createFileSystem(path, enableCache ? hadoopOptionsWithCache : hadoopOptions); + } + + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + return createFileSystem(path, cacheTargetOptions.get(target.name)); + } + + @Override + protected Pair getFileSystemPair( + org.apache.hadoop.fs.Path path, boolean enableCache) throws IOException { + if (!enableCache || cacheRouting == null) { + return super.getFileSystemPair(path, enableCache); + } + CacheTarget target = cacheRouting.targetOf(new Path(path.toUri())); + if (target == null) { + return super.getFileSystemPair(path, false); + } + if (cacheTargetFsMap == null) { + synchronized (this) { + if (cacheTargetFsMap == null) { + cacheTargetFsMap = new ConcurrentHashMap<>(); + } + } + } + String authority = path.toUri().getAuthority(); + String key = target.name + "/" + (authority == null ? "DEFAULT" : authority); + try { + return cacheTargetFsMap.computeIfAbsent(key, k -> createFileSystem(path, target)); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + private Pair createFileSystem( + org.apache.hadoop.fs.Path path, Options options) { final String scheme = path.toUri().getScheme(); final String authority = path.toUri().getAuthority(); - Options options = enableCache ? hadoopOptionsWithCache : hadoopOptions; Supplier> supplier = () -> { Configuration hadoopConf = new Configuration(false); @@ -314,8 +401,14 @@ public synchronized void close() { } } if (!allowCache) { - fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); - fsMap.clear(); + if (fsMap != null) { + fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); + fsMap.clear(); + } + if (cacheTargetFsMap != null) { + cacheTargetFsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); + cacheTargetFsMap.clear(); + } } } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java new file mode 100644 index 000000000000..3fac37529cd7 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.Path; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.FileType; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.JsonSerdeUtil; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link IoCacheRouting}, including the routing cases shared by all clients. */ +public class IoCacheRoutingTest { + + private static final String TABLE_ROOT = "oss://bkt/db1.db/t1"; + private static final Path DATA_FILE = + new Path( + TABLE_ROOT + + "/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc"); + private static final Path MANIFEST = + new Path(TABLE_ROOT + "/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0"); + + @ParameterizedTest(name = "{0}") + @MethodSource("endpointCases") + public void testEndpointOfEachRequest( + String name, Options options, String op, String path, String expect) { + // RESTTokenFileIO puts a client-side dlf.oss-endpoint into fs.oss.endpoint + String override = options.get("dlf.oss-endpoint"); + if (override != null) { + options.set("fs.oss.endpoint", override); + } + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + Options hadoopOptions = fileIO.hadoopOptions(new Path(path), opType(op)); + assertThat(host(hadoopOptions.get("fs.oss.endpoint"))).isEqualTo(host(expect)); + } + + @ParameterizedTest(name = "{0} ({3})") + @MethodSource("fileTypeCases") + public void testCacheableType(String path, Options options, String type, String reason) { + assertThat( + IoCacheRouting.cacheableType( + new Path(path), IoCacheRouting.dataFilePrefixes(options))) + .isEqualTo(type == null ? null : fileType(type)); + } + + @Test + public void testTargetNames() { + Options options = singleTargetOptions(); + options.set("io-cache.targets", " Accel , cluster-1,accel,1x,a_b,,-x"); + options.set("io-cache.target.accel.endpoint", "http://a"); + options.set("io-cache.target.cluster-1.endpoint", "http://c"); + assertThat(IoCacheRouting.create(options).targets()) + .extracting(target -> target.name) + .containsExactly("accel", "cluster-1"); + + Options invalidNames = singleTargetOptions(); + invalidNames.set("io-cache.targets", "Bad_Name"); + assertThat(IoCacheRouting.create(invalidNames)).isNull(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("policyCases") + public void testCachePolicyUsesCompleteCaseInsensitiveTokens( + String policy, boolean readCache, boolean metaCache) { + Options options = singleTargetOptions(); + options.set("io-cache.policy", policy); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + assertThat(host(fileIO.hadoopOptions(DATA_FILE, "read").get("fs.oss.endpoint"))) + .isEqualTo(readCache ? "cache:8080" : "oss-cn-hangzhou.aliyuncs.com"); + assertThat(host(fileIO.hadoopOptions(DATA_FILE, "meta").get("fs.oss.endpoint"))) + .isEqualTo( + metaCache + ? "cache:8080" + : readCache + ? "origin.example.com" + : "oss-cn-hangzhou.aliyuncs.com"); + } + + private static List policyCases() { + return java.util.Arrays.asList( + Arguments.of(" READ , Meta ", true, true), + Arguments.of("read,NONE", false, false), + Arguments.of("thread,metadata", false, false), + Arguments.of("read,nonetheless", true, false)); + } + + @Test + public void testRoutedName() { + String routes = " meta = Accel ; data,BUCKET-INDEX=cluster;=x;junk;file-index=;*=all;x=y"; + assertThat(IoCacheRouting.routedName(routes, FileType.META)).isEqualTo("accel"); + assertThat(IoCacheRouting.routedName(routes, FileType.DATA)).isEqualTo("cluster"); + assertThat(IoCacheRouting.routedName(routes, FileType.BUCKET_INDEX)).isEqualTo("cluster"); + assertThat(IoCacheRouting.routedName(routes, FileType.FILE_INDEX)).isEqualTo("all"); + assertThat(IoCacheRouting.routedName("meta=accel", FileType.DATA)).isNull(); + } + + @Test + public void testTargetSettings() { + Options options = twoTargetOptions(); + options.set("io-cache.target.accel.region", "cn-hangzhou"); + options.set("io-cache.target.cluster.path-style-access", " TRUE "); + options.set("io-cache.target.off.endpoint", ""); + options.set("io-cache.targets", "accel,off,cluster,missing"); + IoCacheRouting routing = IoCacheRouting.create(options); + + assertThat(routing.targets()) + .containsExactly( + new CacheTarget("accel", "https://accel.example.com", false, "cn-hangzhou"), + new CacheTarget("cluster", "http://10.0.0.1:8080", true, null)); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(routing.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(routing.targetOf(new Path(TABLE_ROOT + "/snapshot/snapshot-1"))).isNull(); + + Options bare = singleTargetOptions(); + bare.set("io-cache.endpoint", "cache.example.com"); + assertThat(IoCacheRouting.create(bare).targetOf(DATA_FILE).url) + .isEqualTo("https://cache.example.com"); + } + + @Test + public void testNoIoCacheRouting() { + Options override = singleTargetOptions(); + override.set("dlf.oss-endpoint", "oss-cn-hangzhou.aliyuncs.com"); + assertThat(IoCacheRouting.create(override)).isNull(); + + Options disabled = singleTargetOptions(); + disabled.set("io-cache.enabled", "false"); + assertThat(IoCacheRouting.create(disabled)).isNull(); + + Options writeOnly = singleTargetOptions(); + writeOnly.set("io-cache.policy", "write"); + assertThat(IoCacheRouting.create(writeOnly)).isNull(); + + Options none = singleTargetOptions(); + none.remove("io-cache.endpoint"); + assertThat(IoCacheRouting.create(none)).isNull(); + + // A declared target without an endpoint sends every request to origin. + Options empty = singleTargetOptions(); + empty.set("io-cache.endpoint", ""); + IoCacheRouting routing = IoCacheRouting.create(empty); + assertThat(routing.targets()).isEmpty(); + assertThat(routing.targetOf(DATA_FILE)).isNull(); + assertThat(routing.ossEndpoint()).isEqualTo("https://origin.example.com"); + } + + @Test + public void testOssEndpoint() { + Options options = singleTargetOptions(); + assertThat(IoCacheRouting.create(options).ossEndpoint()) + .isEqualTo("https://origin.example.com"); + options.remove("io-cache.origin.endpoint"); + assertThat(IoCacheRouting.create(options).ossEndpoint()) + .isEqualTo("oss-cn-hangzhou.aliyuncs.com"); + } + + @Test + public void testSerializable() throws Exception { + IoCacheRouting routing = + InstantiationUtil.clone( + IoCacheRouting.create(twoTargetOptions()), getClass().getClassLoader()); + assertThat(routing.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + } + + private static Options singleTargetOptions() { + Options options = new Options(); + options.set("fs.oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + options.set("io-cache.enabled", "true"); + options.set("io-cache.endpoint", "http://cache:8080"); + options.set("io-cache.origin.endpoint", "https://origin.example.com"); + options.set("io-cache.policy", "meta,read"); + return options; + } + + private static Options twoTargetOptions() { + Options options = singleTargetOptions(); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", "https://accel.example.com"); + options.set("io-cache.target.cluster.endpoint", "http://10.0.0.1:8080"); + options.set("io-cache.routes", "meta=accel;data,bucket-index=cluster"); + return options; + } + + private static String host(String endpoint) { + String host = endpoint.contains("://") ? endpoint.split("://", 2)[1] : endpoint; + return host.endsWith("/") ? host.substring(0, host.length() - 1) : host; + } + + private static FileType fileType(String type) { + return FileType.valueOf(type.toUpperCase(Locale.ROOT).replace('-', '_')); + } + + // exists is a status lookup, atomic and two-phase writes are writes; others go to OSS + private static String opType(String op) { + if ("exists".equals(op)) { + return "meta"; + } + return op.endsWith("-write") ? "write" : op; + } + + private static List endpointCases() throws IOException { + JsonNode root = readCases("routing.json"); + List cases = new ArrayList<>(); + for (JsonNode c : root.get("cases")) { + cases.add( + Arguments.of( + c.get("name").asText(), + options(c.get("options")), + c.get("op").asText(), + c.get("path").asText(), + c.get("expect").asText())); + } + assertThat(cases).isNotEmpty(); + return cases; + } + + private static List fileTypeCases() throws IOException { + List cases = new ArrayList<>(); + for (JsonNode c : readCases("routable-types.json").get("cases")) { + JsonNode type = c.get("type"); + cases.add( + Arguments.of( + c.get("path").asText(), + options(c.get("options")), + type.isNull() ? null : type.asText(), + c.get("reason").asText())); + } + assertThat(cases).isNotEmpty(); + return cases; + } + + private static Options options(JsonNode node) { + Options options = new Options(); + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + options.set(field.getKey(), field.getValue().asText()); + } + return options; + } + + private static JsonNode readCases(String name) throws IOException { + try (InputStream in = + IoCacheRoutingTest.class.getClassLoader().getResourceAsStream("io-cache/" + name)) { + assertThat(in).as(name).isNotNull(); + return JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree(in); + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java new file mode 100644 index 000000000000..ca8bbb4c264a --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.Path; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.Pair; + +import com.aliyun.jindodata.api.spec.JdoException; +import com.aliyun.jindodata.common.JindoHadoopSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for the cache endpoints of {@link JindoFileIO}. */ +public class JindoIoCacheRoutingTest { + + private static final String TABLE = "oss://bkt/db1.db/t1"; + private static final String UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; + private static final Path DATA = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-1.orc"); + private static final Path MANIFEST = new Path(TABLE + "/manifest/manifest-1"); + private static final Path SNAPSHOT = new Path(TABLE + "/snapshot/snapshot-2"); + private static final Path LATEST = new Path(TABLE + "/snapshot/LATEST"); + private static final String ACCEL_HOST = "cn-hangzhou-j-internal.oss-data-acc.aliyuncs.com"; + private static final String CLUSTER_HOST = "10.0.0.1:8080"; + private static final String OSS_HOST = "oss-cn-hangzhou-internal.aliyuncs.com"; + private static final String DLF_CACHE_KEY = "fs.oss.dlf-cache.consistent-hash.enabled"; + + private JindoHadoopSystem accelFs; + private JindoHadoopSystem clusterFs; + private JindoHadoopSystem ossFs; + + @BeforeEach + public void before() throws IOException { + accelFs = mockFileSystem(); + clusterFs = mockFileSystem(); + ossFs = mockFileSystem(); + } + + @Test + public void testOptionsOfEachCacheTarget() { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + Options cluster = fileIO.hadoopOptions(DATA, "read"); + assertThat(cluster.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); + assertThat(cluster.get("fs.oss.https.enable")).isEqualTo("false"); + assertThat(cluster.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); + assertThat(cluster.get("fs.oss.region")).isEqualTo("cn-hangzhou"); + assertThat(cluster.get(DLF_CACHE_KEY)).isEqualTo("true"); + assertThat(cluster.containsKey("fs.xengine")).isFalse(); + assertThat(fileIO.hadoopOptions(DATA, "meta")).isSameAs(cluster); + + Options accel = fileIO.hadoopOptions(MANIFEST, "meta"); + assertThat(accel.get("fs.oss.endpoint")).isEqualTo(ACCEL_HOST); + assertThat(accel.get("fs.oss.https.enable")).isEqualTo("true"); + assertThat(accel.get("fs.oss.second.level.domain.enable")).isEqualTo("false"); + assertThat(accel.get("fs.oss.region")).isEqualTo("cn-shanghai"); + + Options oss = fileIO.hadoopOptions(DATA, "write"); + assertThat(oss.get("fs.oss.endpoint")).isEqualTo(OSS_HOST); + assertThat(oss.get("fs.oss.https.enable")).isEqualTo("true"); + assertThat(oss.get("fs.oss.region")).isEqualTo("cn-hangzhou"); + assertThat(oss.containsKey(DLF_CACHE_KEY)).isFalse(); + for (Path path : + new Path[] {SNAPSHOT, LATEST, new Path(TABLE + "/dt=1/bucket-0/000000_0")}) { + assertThat(fileIO.hadoopOptions(path, "read")).as(path.toString()).isSameAs(oss); + } + assertThat(fileIO.hadoopOptions(DATA, "list")).isSameAs(oss); + } + + @Test + public void testOnlyReadsAndStatusUseIoCacheRouting() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path copy = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-2.orc"); + + fileIO.newInputStream(DATA).close(); + fileIO.getFileSize(DATA); + fileIO.getFileStatus(MANIFEST); + fileIO.exists(DATA); + fileIO.exists(MANIFEST); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(clusterFs).getFileStatus(any()); + verify(clusterFs).exists(any()); + verify(accelFs).getFileStatus(any()); + verify(accelFs).exists(any()); + + fileIO.exists(SNAPSHOT); + fileIO.newInputStream(SNAPSHOT).close(); + fileIO.getFileStatus(SNAPSHOT); + fileIO.newOutputStream(DATA, false); + fileIO.listStatus(DATA.getParent()); + fileIO.delete(DATA, false); + fileIO.mkdirs(DATA.getParent()); + fileIO.rename(DATA, copy); + fileIO.copyFile(DATA, copy, true); + verify(ossFs).exists(any()); + verify(ossFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + verify(ossFs, times(2)).create(any(), anyBoolean()); + verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).delete(any(), anyBoolean()); + verify(ossFs).mkdirs(any()); + verify(ossFs).rename(any(), any()); + + // the two-phase write checks a routable data file on OSS, not on its cache target + fileIO.tryToWriteAtomic(new Path(TABLE + "/snapshot/snapshot-3"), "{}"); + fileIO.newTwoPhaseOutputStream(copy, false); + verify(ossFs, times(4)).create(any(), anyBoolean()); + verify(ossFs, times(2)).exists(any()); + verify(ossFs, times(2)).rename(any(), any()); + verify(ossFs).getMpuStore(any()); + + verify(clusterFs).exists(any()); + for (JindoHadoopSystem fs : new JindoHadoopSystem[] {accelFs, clusterFs}) { + verify(fs, never()).create(any(), anyBoolean()); + verify(fs, never()).listStatus(any(org.apache.hadoop.fs.Path.class)); + verify(fs, never()).rename(any(), any()); + } + } + + @Test + public void testCacheTargetErrorsAreThrown() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + IOException unavailable = jindoError(6503, "open failed: 503 Service Unavailable"); + FileNotFoundException notFound = new FileNotFoundException("404"); + when(clusterFs.open(any(org.apache.hadoop.fs.Path.class))).thenThrow(unavailable); + when(accelFs.getFileStatus(any())).thenThrow(notFound); + + assertThatThrownBy(() -> fileIO.newInputStream(DATA)).isSameAs(unavailable); + assertThatThrownBy(() -> fileIO.getFileStatus(MANIFEST)).isSameAs(notFound); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).getFileStatus(any()); + } + + @Test + public void testCacheTargetInitializationErrorsStayChecked() throws IOException { + SocketTimeoutException timeout = new SocketTimeoutException("connect timed out"); + JindoFileIO fileIO = + new TestingJindoFileIO(accelFs, clusterFs, ossFs) { + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + // JindoFileIO wraps an initialization IOException the same way + throw new UncheckedIOException(timeout); + } + }; + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + assertThatThrownBy(() -> fileIO.newInputStream(DATA)).isSameAs(timeout); + assertThatThrownBy(() -> fileIO.getFileStatus(MANIFEST)).isSameAs(timeout); + assertThatThrownBy(() -> fileIO.exists(MANIFEST)).isSameAs(timeout); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testSingleCacheTarget() { + Options options = baseOptions(); + options.set("io-cache.endpoint", "http://" + CLUSTER_HOST); + options.set("io-cache.target.default.path-style-access", "true"); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + + Options read = fileIO.hadoopOptions(DATA, "read"); + assertThat(read.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); + assertThat(read.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); + assertThat(fileIO.hadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.hadoopOptions(DATA, "write").get("fs.oss.endpoint")).isEqualTo(OSS_HOST); + } + + @Test + public void testJindoCacheTakesPrecedence() { + Options options = cacheTargetOptions(); + options.set("fs.jindocache.namespace.rpc.address", "rpc:8101"); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + + assertThat(fileIO.cacheRouting).isNull(); + Options read = fileIO.hadoopOptions(DATA, "read"); + assertThat(read.get("fs.xengine")).isEqualTo("jindocache"); + assertThat(read.get("fs.oss.endpoint")).isEqualTo("http://" + CLUSTER_HOST); + Options write = fileIO.hadoopOptions(DATA, "write"); + assertThat(write.containsKey("fs.xengine")).isFalse(); + // legacy whitelist-path matching: "manifest" is in the default whitelist + assertThat(fileIO.hadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.hadoopOptions(LATEST, "read")).isSameAs(write); + } + + @Test + public void testWithoutIoCacheRouting() { + Options override = cacheTargetOptions(); + override.set("dlf.oss-endpoint", "oss-cn-hangzhou.aliyuncs.com"); + override.set("fs.oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + Options notEnabled = cacheTargetOptions(); + notEnabled.remove("io-cache.enabled"); + Options noPolicy = cacheTargetOptions(); + noPolicy.set("io-cache.policy", "write"); + + for (Options options : new Options[] {override, notEnabled, noPolicy}) { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + assertThat(fileIO.cacheRouting).isNull(); + for (String op : new String[] {"read", "meta", "write"}) { + Options hadoopOptions = fileIO.hadoopOptions(DATA, op); + assertThat(hadoopOptions.get("fs.oss.endpoint")) + .isEqualTo(options.get("fs.oss.endpoint")); + assertThat(hadoopOptions.containsKey("fs.oss.https.enable")).isFalse(); + } + } + } + + @Test + public void testWithEndpoint() { + Options base = new Options(); + base.set("fs.oss.https.enable", "false"); + + Options bare = JindoFileIO.withEndpoint(base, "oss.example.com"); + assertThat(bare.get("fs.oss.endpoint")).isEqualTo("oss.example.com"); + assertThat(bare.get("fs.oss.https.enable")).isEqualTo("false"); + + Options https = JindoFileIO.withEndpoint(base, "HTTPS://cache.example.com:443/"); + assertThat(https.get("fs.oss.endpoint")).isEqualTo("cache.example.com:443"); + assertThat(https.get("fs.oss.https.enable")).isEqualTo("true"); + + Options http = JindoFileIO.withEndpoint(base, "http://10.0.0.1:8080"); + assertThat(http.get("fs.oss.endpoint")).isEqualTo("10.0.0.1:8080"); + assertThat(http.get("fs.oss.https.enable")).isEqualTo("false"); + + assertThat(JindoFileIO.withEndpoint(base, null).toMap()).isEqualTo(base.toMap()); + } + + @Test + public void testFileSystemPerCacheTargetAndBucket() throws IOException { + List created = new ArrayList<>(); + JindoFileIO fileIO = + new JindoFileIO() { + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + created.add(target.name + "/" + path.toUri().getAuthority()); + return Pair.of(mock(JindoHadoopSystem.class), "oss"); + } + }; + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + Pair first = + fileIO.getFileSystemPair(hadoopPath(TABLE + "/manifest/manifest-a"), true); + assertThat(fileIO.getFileSystemPair(hadoopPath(TABLE + "/manifest/manifest-b"), true)) + .isSameAs(first); + fileIO.getFileSystemPair(hadoopPath("oss://other/t/manifest/manifest-c"), true); + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/dt=1/bucket-0/data-" + UUID + "-3.orc"), true); + assertThat(created).containsExactly("accel/bkt", "accel/other", "cluster/bkt"); + } + + @Test + public void testCloseAfterOnlyCacheReads() throws IOException { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = new TestingJindoFileIO(accelFs, clusterFs, ossFs); + fileIO.configure(CatalogContext.create(options)); + fileIO.newInputStream(DATA).close(); + + fileIO.close(); + + verify(clusterFs).close(); + } + + @Test + public void testCloseClosesCacheTargetFileSystems() throws IOException { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + fileIO.fsMap = new ConcurrentHashMap<>(); + fileIO.fsMap.put("bkt", Pair.of(ossFs, "oss")); + fileIO.cacheTargetFsMap = new ConcurrentHashMap<>(); + fileIO.cacheTargetFsMap.put("accel/bkt", Pair.of(accelFs, "oss")); + fileIO.cacheTargetFsMap.put("cluster/bkt", Pair.of(clusterFs, "oss")); + + fileIO.close(); + + for (JindoHadoopSystem fs : new JindoHadoopSystem[] {ossFs, accelFs, clusterFs}) { + verify(fs).close(); + } + assertThat(fileIO.cacheTargetFsMap).isEmpty(); + } + + private static Options baseOptions() { + Options options = new Options(); + options.set("fs.oss.endpoint", "http://" + CLUSTER_HOST); + options.set("fs.oss.accessKeyId", "ak"); + options.set("fs.oss.accessKeySecret", "sk"); + options.set("fs.oss.region", "cn-hangzhou"); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", "https://" + OSS_HOST); + options.set("io-cache.policy", "meta,read"); + options.set(DLF_CACHE_KEY, "true"); + return options; + } + + private static Options cacheTargetOptions() { + Options options = baseOptions(); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", "https://" + ACCEL_HOST); + options.set("io-cache.target.accel.region", "cn-shanghai"); + options.set("io-cache.target.cluster.endpoint", "http://" + CLUSTER_HOST); + options.set("io-cache.target.cluster.path-style-access", "true"); + options.set("io-cache.routes", "meta=accel;data,bucket-index=cluster"); + return options; + } + + private JindoFileIO configuredFileIO(Options options) { + JindoFileIO fileIO = new TestingJindoFileIO(accelFs, clusterFs, ossFs); + fileIO.configure(CatalogContext.create(options)); + return fileIO; + } + + private static IOException jindoError(int code, String message) { + return new IOException(new JdoException(code, message)); + } + + // Hadoop's Path(String) needs commons-lang, which is not on this module's test classpath. + private static org.apache.hadoop.fs.Path hadoopPath(String path) { + return new org.apache.hadoop.fs.Path(URI.create(path)); + } + + private static JindoHadoopSystem mockFileSystem() throws IOException { + JindoHadoopSystem fs = mock(JindoHadoopSystem.class); + when(fs.open(any(org.apache.hadoop.fs.Path.class))) + .thenAnswer(invocation -> new FSDataInputStream(new BytesInput())); + when(fs.create(any(), anyBoolean())).thenReturn(mock(FSDataOutputStream.class)); + when(fs.getFileStatus(any())).thenReturn(mock(FileStatus.class)); + when(fs.rename(any(), any())).thenReturn(true); + return fs; + } + + private static class BytesInput extends ByteArrayInputStream + implements Seekable, PositionedReadable { + + private BytesInput() { + super(new byte[] {1, 2, 3}); + } + + @Override + public void seek(long position) { + pos = (int) position; + } + + @Override + public long getPos() { + return pos; + } + + @Override + public boolean seekToNewSource(long targetPos) { + return false; + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void readFully(long position, byte[] buffer) { + throw new UnsupportedOperationException(); + } + } + + private static class TestingJindoFileIO extends JindoFileIO { + + private final JindoHadoopSystem accelFs; + private final JindoHadoopSystem clusterFs; + private final JindoHadoopSystem ossFs; + + private TestingJindoFileIO( + JindoHadoopSystem accelFs, JindoHadoopSystem clusterFs, JindoHadoopSystem ossFs) { + this.accelFs = accelFs; + this.clusterFs = clusterFs; + this.ossFs = ossFs; + } + + @Override + protected Pair createFileSystem( + org.apache.hadoop.fs.Path path, boolean enableCache) { + assertThat(enableCache).isFalse(); + return Pair.of(ossFs, "oss"); + } + + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + return Pair.of("accel".equals(target.name) ? accelFs : clusterFs, "oss"); + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java index 19a3da7bd533..c09b9ce71153 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java @@ -147,6 +147,22 @@ public void testCacheDisabledWhenPolicyNull() { verifyCacheFlags(fileIO, false, false, false); } + @Test + public void testLegacyPolicyMatchingRemainsUnchanged() { + // JindoCache RPC policies intentionally keep their historical substring matching. + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, "read,nonetheless", true, null)); + verifyCacheFlags(fileIO, false, false, false); + + fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, "thread,metadata", true, null)); + verifyCacheFlags(fileIO, true, true, false); + + fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, " READ , Meta ", true, null)); + verifyCacheFlags(fileIO, false, false, false); + } + @Test public void testCacheWhitelist() { // default config diff --git a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json new file mode 100644 index 000000000000..bb780c885329 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json @@ -0,0 +1,69 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, + {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, + {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, + {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, + {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, + {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, + {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, + {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, + {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, + {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, + {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, + {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, + {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, + {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, + {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} + ] +} diff --git a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json new file mode 100644 index 000000000000..0614fff87ee9 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json @@ -0,0 +1,90 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, + {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, + {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, + {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, + {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, + {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, + {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} + ] +} diff --git a/pom.xml b/pom.xml index ee9ba053991a..05412c5eb3ca 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,8 @@ under the License. paimon-common/src/main/antlr4/** paimon-common/src/test/resources/fmindex-v1-golden.base64 + + paimon-filesystems/paimon-jindo/src/test/resources/io-cache/** paimon-core/src/test/resources/compatibility/** paimon-python/pypaimon/sample/data/** **/*.proto From 6d0ef438ba12422b72865069727d2694dcb6f6dc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 18:27:38 +0800 Subject: [PATCH 02/15] [jindo] Ask OSS whether a file is gone in the FileIO helpers deleteQuietly checks a failed delete with exists(), and readFileUtf8 turns a failed read into FileNotFoundException when exists() says the file is gone. With cache routing, exists() of a cacheable file goes to a cache, which can still hold a file deleted on OSS or report a file missing that OSS has. Both helpers now ask OSS instead. --- .../paimon/jindo/HadoopCompliantFileIO.java | 35 +++++++++++++++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 21 +++++++++++ 2 files changed, 56 insertions(+) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 8d535ebc267f..0cf5a6cfbadf 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -40,6 +40,7 @@ import javax.annotation.Nullable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -248,6 +249,40 @@ public boolean rename(Path src, Path dst) throws IOException { return getFileSystem(hadoopSrc, false).rename(hadoopSrc, hadoopDst); } + @Override + public void deleteQuietly(Path file) { + if (cacheRouting == null) { + FileIO.super.deleteQuietly(file); + return; + } + // a cache may still hold a file that is already gone, so ask OSS + try { + if (!delete(file, false) && existsOnOss(file)) { + LOG.warn("Failed to delete file " + file); + } + } catch (IOException e) { + LOG.warn("Exception occurs when deleting file " + file, e); + } + } + + @Override + public String readFileUtf8(Path path) throws IOException { + try { + return FileIO.super.readFileUtf8(path); + } catch (FileNotFoundException e) { + // FileIO reports a failed read as missing when exists() says so; let OSS decide that + if (cacheRouting != null && e.getCause() instanceof IOException && existsOnOss(path)) { + throw (IOException) e.getCause(); + } + throw e; + } + } + + private boolean existsOnOss(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return getFileSystem(hadoopPath, false).exists(hadoopPath); + } + @Override public void copyFile(Path sourcePath, Path targetPath, boolean overwrite) throws IOException { if (cacheRouting == null) { diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index ca8bbb4c264a..04d5006632d8 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -196,6 +196,27 @@ Pair createFileSystem( verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testHelpersAskOssWhetherAFileIsGone() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + + // the delete failed, OSS no longer has the file, the cache still does + when(clusterFs.exists(any())).thenReturn(true); + fileIO.deleteQuietly(DATA); + verify(ossFs).delete(any(), anyBoolean()); + verify(ossFs).exists(any()); + verify(clusterFs, never()).exists(any()); + + // a read failed on the cache, which no longer has the file, while OSS does + IOException unavailable = jindoError(6503, "read failed: 503 Service Unavailable"); + when(accelFs.open(any(org.apache.hadoop.fs.Path.class))).thenThrow(unavailable); + when(accelFs.exists(any())).thenReturn(false); + when(ossFs.exists(any())).thenReturn(true); + assertThatThrownBy(() -> fileIO.readFileUtf8(MANIFEST)).isSameAs(unavailable); + verify(accelFs).exists(any()); + verify(ossFs, times(2)).exists(any()); + } + @Test public void testSingleCacheTarget() { Options options = baseOptions(); From 6cf5c551d2c485d133ee0471500d4d43bd102d3f Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 19:50:39 +0800 Subject: [PATCH 03/15] [jindo] Route writes of cacheable files when io-cache.policy has write Some cache endpoints, such as an OSS accelerator, also take writes and send them to OSS over their own write path, whose bandwidth adds to the direct one. With write in io-cache.policy, newOutputStream, two-phase multipart writes and hadoopOptions(path, "write") of cacheable files (data files, manifests, indexes) now use the file type's target. The existence check before a two-phase write, atomic writes, renames, deletes, copies, listings and writes of snapshots, hints and temp files stay on OSS. A policy with only write routes writes only. --- .../paimon/jindo/HadoopCompliantFileIO.java | 5 +-- .../apache/paimon/jindo/IoCacheRouting.java | 18 ++++++++--- .../org/apache/paimon/jindo/JindoFileIO.java | 3 +- .../paimon/jindo/IoCacheRoutingTest.java | 9 ++++-- .../paimon/jindo/JindoIoCacheRoutingTest.java | 32 ++++++++++++++++++- .../src/test/resources/io-cache/routing.json | 17 +++++++++- 6 files changed, 73 insertions(+), 11 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 0cf5a6cfbadf..f55ee31acd72 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -118,13 +118,14 @@ public void configure(CatalogContext context) { options.get(IO_CACHE_POLICY)); return; } - // Writes always go to the OSS endpoint, so writeCacheEnabled stays false. metaCacheEnabled = cacheRouting.metaCacheEnabled(); readCacheEnabled = cacheRouting.readCacheEnabled(); + writeCacheEnabled = cacheRouting.writeCacheEnabled(); LOG.info( - "Cache endpoints enabled: meta cache enabled {}, read cache enabled {}, {}", + "Cache endpoints enabled: meta {}, read {}, write {}, {}", metaCacheEnabled, readCacheEnabled, + writeCacheEnabled, cacheRouting); } else { // Keep legacy JindoCache policy matching unchanged; endpoint routing uses exact tokens. diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index e8caf99f4059..7635d263fe91 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -50,7 +50,7 @@ /** * Selects cache targets for immutable Paimon files using {@code io-cache.targets}, {@code * io-cache.routes} and {@code io-cache.whitelist}. The endpoint mode also carries the normalized - * {@code io-cache.policy} read and metadata switches. + * {@code io-cache.policy} read, metadata and write switches. */ final class IoCacheRouting implements Serializable { @@ -58,6 +58,7 @@ final class IoCacheRouting implements Serializable { private static final String META_CACHE_ENABLED_TAG = "meta"; private static final String READ_CACHE_ENABLED_TAG = "read"; + private static final String WRITE_CACHE_ENABLED_TAG = "write"; private static final String DISABLE_CACHE_TAG = "none"; private static final Pattern TARGET_NAME = Pattern.compile("[a-z][a-z0-9-]*"); @@ -76,6 +77,7 @@ final class IoCacheRouting implements Serializable { private final List dataFilePrefixes; private final boolean metaCacheEnabled; private final boolean readCacheEnabled; + private final boolean writeCacheEnabled; private IoCacheRouting( @Nullable String ossEndpoint, @@ -83,13 +85,15 @@ private IoCacheRouting( Map targetByType, List dataFilePrefixes, boolean metaCacheEnabled, - boolean readCacheEnabled) { + boolean readCacheEnabled, + boolean writeCacheEnabled) { this.ossEndpoint = ossEndpoint; this.targets = targets; this.targetByType = targetByType; this.dataFilePrefixes = dataFilePrefixes; this.metaCacheEnabled = metaCacheEnabled; this.readCacheEnabled = readCacheEnabled; + this.writeCacheEnabled = writeCacheEnabled; } /** Creates endpoint routing from the options, or null when routing is not enabled. */ @@ -101,7 +105,8 @@ static IoCacheRouting create(Options options) { Set policy = parsePolicy(options.get(IO_CACHE_POLICY)); boolean metaCache = policy.contains(META_CACHE_ENABLED_TAG); boolean readCache = policy.contains(READ_CACHE_ENABLED_TAG); - if (policy.contains(DISABLE_CACHE_TAG) || !(metaCache || readCache)) { + boolean writeCache = policy.contains(WRITE_CACHE_ENABLED_TAG); + if (policy.contains(DISABLE_CACHE_TAG) || !(metaCache || readCache || writeCache)) { return null; } // a client-side dlf.oss-endpoint replaces every endpoint the token vends @@ -144,7 +149,8 @@ static IoCacheRouting create(Options options) { targetByType, dataFilePrefixes(options), metaCache, - readCache); + readCache, + writeCache); } private static Set parsePolicy(@Nullable String value) { @@ -163,6 +169,10 @@ boolean readCacheEnabled() { return readCacheEnabled; } + boolean writeCacheEnabled() { + return writeCacheEnabled; + } + /** The OSS endpoint used by requests that do not use a cache target. */ @Nullable String ossEndpoint() { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index f8ab0e55e66b..4488c3b8aa73 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -247,7 +247,8 @@ public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite if (!overwrite && exists) { throw new IOException("File " + path + " already exists."); } - Pair pair = getFileSystemPair(hadoopPath, false); + boolean viaTarget = cacheRouting != null && writeCacheEnabled && shouldCache(path); + Pair pair = getFileSystemPair(hadoopPath, viaTarget); JindoHadoopSystem fs = pair.getKey(); JindoMpuStore mpuStore = fs.getMpuStore(hadoopPath); if (mpuStore == null) { diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index 3fac37529cd7..e6c3ea55bbe0 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -165,7 +165,9 @@ public void testNoIoCacheRouting() { Options writeOnly = singleTargetOptions(); writeOnly.set("io-cache.policy", "write"); - assertThat(IoCacheRouting.create(writeOnly)).isNull(); + IoCacheRouting writes = IoCacheRouting.create(writeOnly); + assertThat(writes.writeCacheEnabled()).isTrue(); + assertThat(writes.readCacheEnabled() || writes.metaCacheEnabled()).isFalse(); Options none = singleTargetOptions(); none.remove("io-cache.endpoint"); @@ -227,11 +229,14 @@ private static FileType fileType(String type) { return FileType.valueOf(type.toUpperCase(Locale.ROOT).replace('-', '_')); } - // exists is a status lookup, atomic and two-phase writes are writes; others go to OSS + // exists is a status lookup; an atomic write stages a temp file and renames it on OSS private static String opType(String op) { if ("exists".equals(op)) { return "meta"; } + if ("atomic-write".equals(op)) { + return "rename"; + } return op.endsWith("-write") ? "write" : op; } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 04d5006632d8..ceec10d8ecf6 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -196,6 +196,36 @@ Pair createFileSystem( verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testWritesFollowTheRouteWithWritePolicy() throws IOException { + Options options = cacheTargetOptions(); + options.set("io-cache.policy", "meta,read,write"); + JindoFileIO fileIO = configuredFileIO(options); + Path next = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-2.orc"); + + fileIO.newOutputStream(DATA, false); + fileIO.newOutputStream(MANIFEST, false); + fileIO.newOutputStream(SNAPSHOT, false); + fileIO.newTwoPhaseOutputStream(next, false); + assertThat(fileIO.hadoopOptions(DATA, "write").get("fs.oss.endpoint")) + .isEqualTo(CLUSTER_HOST); + verify(clusterFs).create(any(), anyBoolean()); + verify(accelFs).create(any(), anyBoolean()); + verify(clusterFs).getMpuStore(any()); + // the snapshot, the existence check before the two-phase write and the temp file of its + // rename fallback (no multipart upload here) stay on OSS + verify(ossFs, times(2)).create(any(), anyBoolean()); + verify(ossFs).exists(any()); + verify(clusterFs, never()).exists(any()); + + fileIO.delete(DATA, false); + fileIO.rename(DATA, next); + fileIO.listStatus(DATA.getParent()); + verify(ossFs).delete(any(), anyBoolean()); + verify(ossFs).rename(any(), any()); + verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); + } + @Test public void testHelpersAskOssWhetherAFileIsGone() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); @@ -258,7 +288,7 @@ public void testWithoutIoCacheRouting() { Options notEnabled = cacheTargetOptions(); notEnabled.remove("io-cache.enabled"); Options noPolicy = cacheTargetOptions(); - noPolicy.set("io-cache.policy", "write"); + noPolicy.set("io-cache.policy", "prefetch"); for (Options options : new Options[] {override, notEnabled, noPolicy}) { JindoFileIO fileIO = new JindoFileIO(); diff --git a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json index 0614fff87ee9..afd4b88eb836 100644 --- a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json +++ b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json @@ -33,7 +33,22 @@ {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, From 30a7d2539b3f8d729af28afcf6c362cc5f1cf264 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 23:45:12 +0800 Subject: [PATCH 04/15] [jindo] Keep the FileIO defaults of deleteQuietly, readFileUtf8 and copyFile The routing FileIO no longer overrides these helpers. - deleteQuietly only logs a warning when exists() still finds a deleted file, so asking a cache changes a log line, not a result. - readFileUtf8 reports a failed read as not found only when exists() says the file is gone. A cacheable file is never probed before it is written, so a cache does not report an existing one as missing. - copyFile is a read plus a write, and each is routed like any other read or write now that writes follow io-cache.policy. --- .../paimon/jindo/HadoopCompliantFileIO.java | 51 ------------------- .../paimon/jindo/JindoIoCacheRoutingTest.java | 28 ++-------- 2 files changed, 3 insertions(+), 76 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index f55ee31acd72..195445108927 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -40,7 +40,6 @@ import javax.annotation.Nullable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -250,56 +249,6 @@ public boolean rename(Path src, Path dst) throws IOException { return getFileSystem(hadoopSrc, false).rename(hadoopSrc, hadoopDst); } - @Override - public void deleteQuietly(Path file) { - if (cacheRouting == null) { - FileIO.super.deleteQuietly(file); - return; - } - // a cache may still hold a file that is already gone, so ask OSS - try { - if (!delete(file, false) && existsOnOss(file)) { - LOG.warn("Failed to delete file " + file); - } - } catch (IOException e) { - LOG.warn("Exception occurs when deleting file " + file, e); - } - } - - @Override - public String readFileUtf8(Path path) throws IOException { - try { - return FileIO.super.readFileUtf8(path); - } catch (FileNotFoundException e) { - // FileIO reports a failed read as missing when exists() says so; let OSS decide that - if (cacheRouting != null && e.getCause() instanceof IOException && existsOnOss(path)) { - throw (IOException) e.getCause(); - } - throw e; - } - } - - private boolean existsOnOss(Path path) throws IOException { - org.apache.hadoop.fs.Path hadoopPath = path(path); - return getFileSystem(hadoopPath, false).exists(hadoopPath); - } - - @Override - public void copyFile(Path sourcePath, Path targetPath, boolean overwrite) throws IOException { - if (cacheRouting == null) { - FileIO.super.copyFile(sourcePath, targetPath, overwrite); - return; - } - // copies read the source from the OSS endpoint, not from a cache endpoint - org.apache.hadoop.fs.Path hadoopSrc = path(sourcePath); - org.apache.hadoop.fs.Path hadoopDst = path(targetPath); - try (FSDataInputStream in = getFileSystem(hadoopSrc, false).open(hadoopSrc); - FSDataOutputStream out = - getFileSystem(hadoopDst, false).create(hadoopDst, overwrite)) { - IOUtils.copyBytes(in, out, 4096); - } - } - protected org.apache.hadoop.fs.Path path(Path path) { URI uri = path.toUri(); if ("oss".equals(uri.getScheme()) && uri.getUserInfo() != null) { diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index ceec10d8ecf6..5d2bec1aa3ea 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -136,11 +136,10 @@ public void testOnlyReadsAndStatusUseIoCacheRouting() throws IOException { fileIO.delete(DATA, false); fileIO.mkdirs(DATA.getParent()); fileIO.rename(DATA, copy); - fileIO.copyFile(DATA, copy, true); verify(ossFs).exists(any()); - verify(ossFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); verify(ossFs).getFileStatus(any()); - verify(ossFs, times(2)).create(any(), anyBoolean()); + verify(ossFs).create(any(), anyBoolean()); verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); verify(ossFs).delete(any(), anyBoolean()); verify(ossFs).mkdirs(any()); @@ -149,7 +148,7 @@ public void testOnlyReadsAndStatusUseIoCacheRouting() throws IOException { // the two-phase write checks a routable data file on OSS, not on its cache target fileIO.tryToWriteAtomic(new Path(TABLE + "/snapshot/snapshot-3"), "{}"); fileIO.newTwoPhaseOutputStream(copy, false); - verify(ossFs, times(4)).create(any(), anyBoolean()); + verify(ossFs, times(3)).create(any(), anyBoolean()); verify(ossFs, times(2)).exists(any()); verify(ossFs, times(2)).rename(any(), any()); verify(ossFs).getMpuStore(any()); @@ -226,27 +225,6 @@ public void testWritesFollowTheRouteWithWritePolicy() throws IOException { verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); } - @Test - public void testHelpersAskOssWhetherAFileIsGone() throws IOException { - JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); - - // the delete failed, OSS no longer has the file, the cache still does - when(clusterFs.exists(any())).thenReturn(true); - fileIO.deleteQuietly(DATA); - verify(ossFs).delete(any(), anyBoolean()); - verify(ossFs).exists(any()); - verify(clusterFs, never()).exists(any()); - - // a read failed on the cache, which no longer has the file, while OSS does - IOException unavailable = jindoError(6503, "read failed: 503 Service Unavailable"); - when(accelFs.open(any(org.apache.hadoop.fs.Path.class))).thenThrow(unavailable); - when(accelFs.exists(any())).thenReturn(false); - when(ossFs.exists(any())).thenReturn(true); - assertThatThrownBy(() -> fileIO.readFileUtf8(MANIFEST)).isSameAs(unavailable); - verify(accelFs).exists(any()); - verify(ossFs, times(2)).exists(any()); - } - @Test public void testSingleCacheTarget() { Options options = baseOptions(); From f2e0380b453cac69944a7aea829e49a983ab179c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 23:45:12 +0800 Subject: [PATCH 05/15] [jindo] Write the io-cache routing test cases inline Replace the JSON test vectors with assertions grouped by topic in IoCacheRoutingTest, and drop their license exclusion from the root pom. --- .../paimon/jindo/IoCacheRoutingTest.java | 411 ++++++++++++++---- .../resources/io-cache/routable-types.json | 69 --- .../src/test/resources/io-cache/routing.json | 105 ----- pom.xml | 2 - 4 files changed, 323 insertions(+), 264 deletions(-) delete mode 100644 paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json delete mode 100644 paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index e6c3ea55bbe0..bf02eac80f4e 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -24,58 +24,286 @@ import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; import org.apache.paimon.utils.InstantiationUtil; -import org.apache.paimon.utils.JsonSerdeUtil; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; +import java.util.Arrays; import java.util.List; -import java.util.Locale; -import java.util.Map; +import static org.apache.paimon.utils.FileType.BUCKET_INDEX; +import static org.apache.paimon.utils.FileType.DATA; +import static org.apache.paimon.utils.FileType.FILE_INDEX; +import static org.apache.paimon.utils.FileType.GLOBAL_INDEX; +import static org.apache.paimon.utils.FileType.META; import static org.assertj.core.api.Assertions.assertThat; -/** Tests for {@link IoCacheRouting}, including the routing cases shared by all clients. */ +/** Tests for {@link IoCacheRouting}. */ public class IoCacheRoutingTest { private static final String TABLE_ROOT = "oss://bkt/db1.db/t1"; + private static final String UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; private static final Path DATA_FILE = - new Path( - TABLE_ROOT - + "/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc"); - private static final Path MANIFEST = - new Path(TABLE_ROOT + "/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0"); + new Path(TABLE_ROOT + "/dt=2026-09-30/bucket-0/data-" + UUID + "-0.orc"); + private static final Path MANIFEST = new Path(TABLE_ROOT + "/manifest/manifest-" + UUID + "-0"); + + // paths below are relative to TABLE_ROOT, and {uuid} stands for UUID + private static final String DATA_PATH = "dt=1/bucket-0/data-{uuid}-0.parquet"; + private static final String MANIFEST_PATH = "manifest/manifest-{uuid}-0"; + private static final String INDEX_PATH = "index/index-{uuid}-0"; + private static final String GLOBAL_INDEX_PATH = "index/btree-global-index-{uuid}.index"; + private static final String TEMP_PATH = "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp"; + private static final String SNAPSHOT_PATH = "snapshot/snapshot-12"; + private static final String LATEST_PATH = "snapshot/LATEST"; + + private static final String OSS = "https://oss-cn-hangzhou-internal.aliyuncs.com"; + private static final String CACHE = "http://cache.example.com"; + private static final String ACCEL = "https://accelerator.example.com"; + private static final String CLUSTER = "http://cluster.example.com"; + private static final String WRITE = "io-cache.policy=meta,read,write"; - @ParameterizedTest(name = "{0}") - @MethodSource("endpointCases") - public void testEndpointOfEachRequest( - String name, Options options, String op, String path, String expect) { - // RESTTokenFileIO puts a client-side dlf.oss-endpoint into fs.oss.endpoint - String override = options.get("dlf.oss-endpoint"); - if (override != null) { - options.set("fs.oss.endpoint", override); + @Test + public void testOneCacheTarget() { + Options options = single(); + assertEndpoint(options, "read", DATA_PATH, CACHE); + assertEndpoint(options, "meta", DATA_PATH, CACHE); + assertEndpoint(options, "exists", DATA_PATH, CACHE); + assertEndpoint(options, "read", MANIFEST_PATH, CACHE); + assertEndpoint(options, "read", "manifest/manifest-list-{uuid}-1", CACHE); + assertEndpoint(options, "read", "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE); + assertEndpoint(options, "read", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "meta", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "exists", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "read", LATEST_PATH, OSS); + assertEndpoint(options, "exists", LATEST_PATH, OSS); + assertEndpoint(options, "read", TEMP_PATH, OSS); + assertEndpoint(options, "read", "dt=1/bucket-0/000000_0", OSS); + assertEndpoint(options, "read", "dls://bkt/db1.db/t1/" + DATA_PATH, OSS); + // the whitelist has no index + assertEndpoint(options, "read", INDEX_PATH, OSS); + assertEndpoint(options, "write", DATA_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, DATA_PATH, OSS); } - JindoFileIO fileIO = new JindoFileIO(); - fileIO.configure(CatalogContext.create(options)); - Options hadoopOptions = fileIO.hadoopOptions(new Path(path), opType(op)); - assertThat(host(hadoopOptions.get("fs.oss.endpoint"))).isEqualTo(host(expect)); } - @ParameterizedTest(name = "{0} ({3})") - @MethodSource("fileTypeCases") - public void testCacheableType(String path, Options options, String type, String reason) { - assertThat( - IoCacheRouting.cacheableType( - new Path(path), IoCacheRouting.dataFilePrefixes(options))) - .isEqualTo(type == null ? null : fileType(type)); + @Test + public void testPolicyAndWhitelist() { + assertEndpoint(single("-io-cache.enabled"), "read", DATA_PATH, CACHE); + assertEndpoint(single("io-cache.enabled=false"), "exists", DATA_PATH, CACHE); + assertEndpoint(single("-io-cache.policy"), "exists", DATA_PATH, CACHE); + assertEndpoint(single("io-cache.policy=none"), "exists", DATA_PATH, CACHE); + + Options readOnly = single("io-cache.policy=read"); + assertEndpoint(readOnly, "read", DATA_PATH, CACHE); + assertEndpoint(readOnly, "meta", DATA_PATH, OSS); + assertEndpoint(readOnly, "exists", DATA_PATH, OSS); + Options writeOnly = single("io-cache.policy=write"); + assertEndpoint(writeOnly, "write", DATA_PATH, CACHE); + assertEndpoint(writeOnly, "exists", DATA_PATH, OSS); + + assertEndpoint(single("-io-cache.whitelist"), "read", INDEX_PATH, CACHE); + assertEndpoint(single("io-cache.whitelist=*"), "read", GLOBAL_INDEX_PATH, CACHE); + assertEndpoint(single("io-cache.whitelist=*"), "read", DATA_PATH + ".index", CACHE); + } + + @Test + public void testEndpoints() { + assertEndpoint(single("io-cache.endpoint= "), "read", DATA_PATH, OSS); + assertEndpoint( + single("io-cache.endpoint=", "io-cache.routes=*=default"), "read", DATA_PATH, OSS); + // the origin defaults to fs.oss.endpoint + String oss = "fs.oss.endpoint=" + OSS; + assertEndpoint(single(oss, "-io-cache.origin.endpoint"), "list", DATA_PATH, OSS); + assertEndpoint(single(oss, "io-cache.origin.endpoint= "), "list", DATA_PATH, OSS); + // a client-side endpoint turns routing off + String override = "https://oss-cn-hangzhou.aliyuncs.com"; + assertEndpoint(single("dlf.oss-endpoint=" + override), "read", DATA_PATH, override); + } + + @Test + public void testWritePolicy() { + Options options = single(WRITE); + assertEndpoint(options, "write", DATA_PATH, CACHE); + assertEndpoint(options, "write", MANIFEST_PATH, CACHE); + assertEndpoint(options, "write", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "write", LATEST_PATH, OSS); + assertEndpoint(options, "write", TEMP_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, DATA_PATH, OSS); + } + assertEndpoint(single(WRITE, "io-cache.whitelist=meta"), "write", DATA_PATH, OSS); + + assertEndpoint(multi(WRITE), "write", DATA_PATH, CLUSTER); + assertEndpoint(multi(WRITE), "write", MANIFEST_PATH, ACCEL); + } + + @Test + public void testTwoCacheTargets() { + Options options = multi(); + assertEndpoint(options, "read", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "meta", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "exists", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "read", DATA_PATH, CLUSTER); + assertEndpoint(options, "exists", DATA_PATH, CLUSTER); + assertEndpoint(options, "read", INDEX_PATH, CLUSTER); + assertEndpoint(options, "read", GLOBAL_INDEX_PATH, CLUSTER); + assertEndpoint(options, "read", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "write", MANIFEST_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, MANIFEST_PATH, OSS); + } + assertEndpoint(multi("io-cache.whitelist=meta"), "read", DATA_PATH, OSS); + } + + @Test + public void testTargetsAndRoutes() { + // without routes the first target with an endpoint takes every type + String noRoutes = "-io-cache.routes"; + assertEndpoint(multi(noRoutes), "read", DATA_PATH, ACCEL); + assertEndpoint(multi(noRoutes, "io-cache.endpoint=" + CACHE), "read", DATA_PATH, ACCEL); + assertEndpoint( + multi(noRoutes, "-io-cache.target.accel.endpoint"), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi(noRoutes, "io-cache.target.accel.endpoint= "), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi(noRoutes, "io-cache.targets=Bad_Name,cluster"), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi("io-cache.target.cluster.endpoint= " + CLUSTER + " "), + "read", + DATA_PATH, + CLUSTER); + + // a type without a usable route goes to the origin + assertEndpoint(multi("io-cache.routes=data=x"), "read", DATA_PATH, OSS); + assertEndpoint(multi("-io-cache.target.cluster.endpoint"), "read", DATA_PATH, OSS); + String metaAndData = "io-cache.routes=meta=accel;data=cluster"; + assertEndpoint(multi(metaAndData), "read", DATA_PATH + ".index", OSS); + + assertEndpoint(multi("io-cache.routes=*=cluster"), "read", MANIFEST_PATH, CLUSTER); + String firstWins = "io-cache.routes=data=accel;data,meta=cluster"; + assertEndpoint(multi(firstWins), "read", DATA_PATH, ACCEL); + String malformed = "io-cache.routes==cluster;meta=accel;junk"; + assertEndpoint(multi(malformed), "read", MANIFEST_PATH, ACCEL); + assertEndpoint(multi(malformed), "read", DATA_PATH, OSS); + } + + @Test + public void testDataFilesNamedLikeMetadata() { + Options manifest = multi("data-file.prefix=manifest-"); + assertEndpoint(manifest, "read", "dt=1/bucket-0/manifest-{uuid}-0.orc", CLUSTER); + assertEndpoint(manifest, "read", "dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", CLUSTER); + assertEndpoint(manifest, "read", MANIFEST_PATH, ACCEL); + assertEndpoint(manifest, "read", MANIFEST_PATH + ".avro.sidecar", ACCEL); + Options snapshot = multi("data-file.prefix=snapshot-"); + assertEndpoint(snapshot, "read", "dt=1/bucket-0/snapshot-{uuid}-0.orc", CLUSTER); + Options stat = multi("data-file.prefix=stat-"); + assertEndpoint(stat, "read", "dt=1/bucket-0/stat-{uuid}-0.orc", CLUSTER); + Options index = + multi("io-cache.routes=data=cluster;bucket-index=accel", "data-file.prefix=index-"); + assertEndpoint(index, "read", "dt=1/bucket-0/index-{uuid}-0.orc", CLUSTER); + assertEndpoint(index, "read", INDEX_PATH, ACCEL); + + String custom = "dt=1/bucket-0/custom-{uuid}-0.orc"; + assertEndpoint(single(), "read", custom, OSS); + assertEndpoint(single("data-file.prefix=custom-"), "read", custom, CACHE); + } + + @Test + public void testCacheableType() { + // named by sequence id or rewritten in place + for (String path : + Arrays.asList( + "snapshot/snapshot-12", + "snapshot/LATEST", + "snapshot/EARLIEST", + "branch/branch-dev/snapshot/LATEST", + "schema/schema-3", + "changelog/changelog-5", + "changelog/LATEST", + "tag/tag-2026-09-30", + "tag/tag-success-file/t1_SUCCESS", + "consumer/consumer-job1", + "service/service-primary-key-lookup", + "dt=1/_SUCCESS", + "oss://bkt/bucket-0/db/t/changelog/changelog-5")) { + assertType(path, null); + } + // temporary or not written by Paimon + for (String path : + Arrays.asList( + "snapshot/.snapshot-13.{uuid}.tmp", + "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp-{uuid}", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp.{uuid}", + "metadata/version-hint.text", + "metadata/v3.metadata.json", + "dt=1/bucket-0/000000_0", + "dt=1/part-00000-1a2b.snappy.parquet", + "dt=1/bucket-0/data-1.parquet", + "dt=1/bucket-0/custom-{uuid}-0.orc", + "README")) { + assertType(path, null); + } + + assertType("manifest/manifest-{uuid}-0", META); + assertType("manifest/manifest-list-{uuid}-1", META); + assertType("manifest/index-manifest-{uuid}-0", META); + assertType("manifest/manifest-{uuid}-0.avro.sidecar", META); + assertType("statistics/stat-{uuid}-0", META); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/changelog-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.blob", DATA); + assertType("dt=1/8b1f7c2e/data-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.index", FILE_INDEX); + assertType("index/index-{uuid}-0", BUCKET_INDEX); + assertType("index/btree-global-index-{uuid}.index", GLOBAL_INDEX); + assertType("index/lumina-global-index-{uuid}.index", GLOBAL_INDEX); + } + + @Test + public void testCacheableTypeWithFilePrefixes() { + assertType("dt=1/bucket-0/custom-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); + assertType("dt=1/bucket-0/data-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); + assertType("dt=1/bucket-0/cl-{uuid}-0.orc", DATA, "changelog-file.prefix=cl-"); + assertType("dt=1/bucket-0/ unknown.parquet", null, "data-file.prefix= "); + + // the name decides, so data files may be named like metadata + String manifest = "data-file.prefix=manifest-"; + assertType("dt=1/bucket-0/manifest-{uuid}-0.orc", DATA, manifest); + assertType("dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", DATA, manifest); + assertType("dt=1/bucket-0/manifest-{uuid}-0.orc.index", FILE_INDEX, manifest); + assertType("manifest/manifest-{uuid}-0", META, manifest); + assertType("manifest/manifest-{uuid}-0.avro.sidecar", META, manifest); + String index = "data-file.prefix=index-"; + assertType("dt=1/bucket-0/index-{uuid}-0.orc", DATA, index); + assertType("bucket-postpone/index-{uuid}-0.orc", DATA, index); + assertType("index/index-{uuid}-0", BUCKET_INDEX, index); + assertType("dt=1/bucket-0/index-{uuid}-0", BUCKET_INDEX, index); + String stat = "data-file.prefix=stat-"; + assertType("dt=1/bucket-0/stat-{uuid}-0.orc", DATA, stat); + assertType("postpone/stat-{uuid}-0.orc", DATA, stat); + assertType("statistics/stat-{uuid}-0", META, stat); + String snapshot = "data-file.prefix=snapshot-"; + assertType("dt=1/bucket-0/snapshot-{uuid}-0.orc", DATA, snapshot); + assertType("dt=1/bucket-0/snapshot-{uuid}-0.orc.index", FILE_INDEX, snapshot); + assertType("dt=1/bucket-0/schema-{uuid}-0.orc", DATA, "data-file.prefix=schema-"); + assertType( + "dt=1/bucket-0/global-index-{uuid}-0.orc.index", + FILE_INDEX, + "data-file.prefix=global-index-"); + + // names rewritten in place never use a cache, whatever the prefix + assertType("dt=1/bucket-0/tag-{uuid}-0.orc", null, "data-file.prefix=tag-"); + assertType("dt=1/bucket-0/consumer-{uuid}-0.orc", null, "data-file.prefix=consumer-"); + String warehouse = "oss://bkt/warehouse/bucket-0/db/t1/"; + assertType(warehouse + "tag/tag-file", null, "data-file.prefix=tag-"); + assertType(warehouse + "consumer/consumer-file", null, "data-file.prefix=consumer-"); + assertType(warehouse + "service/service-file", null, "data-file.prefix=service-"); + assertType(warehouse + "metadata/version-hint.text", null, "data-file.prefix=version-"); + assertType(warehouse + "branch/branch-feature", null, "data-file.prefix=branch-"); } @Test @@ -113,7 +341,7 @@ public void testCachePolicyUsesCompleteCaseInsensitiveTokens( } private static List policyCases() { - return java.util.Arrays.asList( + return Arrays.asList( Arguments.of(" READ , Meta ", true, true), Arguments.of("read,NONE", false, false), Arguments.of("thread,metadata", false, false), @@ -220,72 +448,79 @@ private static Options twoTargetOptions() { return options; } - private static String host(String endpoint) { - String host = endpoint.contains("://") ? endpoint.split("://", 2)[1] : endpoint; - return host.endsWith("/") ? host.substring(0, host.length() - 1) : host; + /** One cache target, which older clients also get as fs.oss.endpoint. */ + private static Options single(String... changes) { + Options options = new Options(); + options.set("fs.oss.endpoint", CACHE); + options.set("io-cache.enabled", "true"); + options.set("io-cache.endpoint", CACHE); + options.set("io-cache.origin.endpoint", OSS); + options.set("io-cache.policy", "meta,read"); + options.set("io-cache.whitelist", "meta,data"); + return with(options, changes); } - private static FileType fileType(String type) { - return FileType.valueOf(type.toUpperCase(Locale.ROOT).replace('-', '_')); + /** Metadata on an accelerator, data and indexes on a cache cluster. */ + private static Options multi(String... changes) { + Options options = new Options(); + options.set("fs.oss.endpoint", ACCEL); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", OSS); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", ACCEL); + options.set("io-cache.target.accel.region", "cn-hangzhou"); + options.set("io-cache.target.cluster.endpoint", CLUSTER); + options.set("io-cache.target.cluster.path-style-access", "true"); + options.set("io-cache.policy", "meta,read"); + options.set("io-cache.whitelist", "*"); + options.set( + "io-cache.routes", "meta=accel;data,bucket-index,global-index,file-index=cluster"); + return with(options, changes); } - // exists is a status lookup; an atomic write stages a temp file and renames it on OSS - private static String opType(String op) { - if ("exists".equals(op)) { - return "meta"; - } - if ("atomic-write".equals(op)) { - return "rename"; + // "key=value" sets an option and "-key" removes it + private static Options with(Options options, String... changes) { + for (String change : changes) { + if (change.startsWith("-")) { + options.remove(change.substring(1)); + } else { + int eq = change.indexOf('='); + options.set(change.substring(0, eq), change.substring(eq + 1)); + } } - return op.endsWith("-write") ? "write" : op; + return options; } - private static List endpointCases() throws IOException { - JsonNode root = readCases("routing.json"); - List cases = new ArrayList<>(); - for (JsonNode c : root.get("cases")) { - cases.add( - Arguments.of( - c.get("name").asText(), - options(c.get("options")), - c.get("op").asText(), - c.get("path").asText(), - c.get("expect").asText())); + private static void assertEndpoint(Options options, String op, String path, String expect) { + Options copy = new Options(options.toMap()); + // RESTTokenFileIO puts a client-side dlf.oss-endpoint into fs.oss.endpoint + String override = copy.get("dlf.oss-endpoint"); + if (override != null) { + copy.set("fs.oss.endpoint", override); } - assertThat(cases).isNotEmpty(); - return cases; + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(copy)); + // exists is a status lookup + String opType = "exists".equals(op) ? "meta" : op; + assertThat(host(fileIO.hadoopOptions(path(path), opType).get("fs.oss.endpoint"))) + .as("%s %s with %s", op, path, options.toMap()) + .isEqualTo(host(expect)); } - private static List fileTypeCases() throws IOException { - List cases = new ArrayList<>(); - for (JsonNode c : readCases("routable-types.json").get("cases")) { - JsonNode type = c.get("type"); - cases.add( - Arguments.of( - c.get("path").asText(), - options(c.get("options")), - type.isNull() ? null : type.asText(), - c.get("reason").asText())); - } - assertThat(cases).isNotEmpty(); - return cases; + private static void assertType(String path, FileType type, String... changes) { + List prefixes = IoCacheRouting.dataFilePrefixes(with(new Options(), changes)); + assertThat(IoCacheRouting.cacheableType(path(path), prefixes)) + .as("%s with %s", path, Arrays.toString(changes)) + .isEqualTo(type); } - private static Options options(JsonNode node) { - Options options = new Options(); - Iterator> fields = node.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - options.set(field.getKey(), field.getValue().asText()); - } - return options; + private static Path path(String path) { + String resolved = path.replace("{uuid}", UUID); + return new Path(resolved.contains("://") ? resolved : TABLE_ROOT + "/" + resolved); } - private static JsonNode readCases(String name) throws IOException { - try (InputStream in = - IoCacheRoutingTest.class.getClassLoader().getResourceAsStream("io-cache/" + name)) { - assertThat(in).as(name).isNotNull(); - return JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree(in); - } + private static String host(String endpoint) { + String host = endpoint.contains("://") ? endpoint.split("://", 2)[1] : endpoint; + return host.endsWith("/") ? host.substring(0, host.length() - 1) : host; } } diff --git a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json deleted file mode 100644 index bb780c885329..000000000000 --- a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routable-types.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, - {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, - {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, - {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, - {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, - {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, - {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, - {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, - {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, - {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, - {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, - {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, - {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, - {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, - {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} - ] -} diff --git a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json b/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json deleted file mode 100644 index afd4b88eb836..000000000000 --- a/paimon-filesystems/paimon-jindo/src/test/resources/io-cache/routing.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, - {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, - {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, - {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, - {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, - {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, - {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} - ] -} diff --git a/pom.xml b/pom.xml index 05412c5eb3ca..ee9ba053991a 100644 --- a/pom.xml +++ b/pom.xml @@ -675,8 +675,6 @@ under the License. paimon-common/src/main/antlr4/** paimon-common/src/test/resources/fmindex-v1-golden.base64 - - paimon-filesystems/paimon-jindo/src/test/resources/io-cache/** paimon-core/src/test/resources/compatibility/** paimon-python/pypaimon/sample/data/** **/*.proto From 421917d08431ebbcb5125c2508e7d401f1bb3e85 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 15:06:03 +0800 Subject: [PATCH 06/15] [jindo] Route exists to a target only with exists in io-cache.policy exists answers whether a file is still there: commit recovery checks the files it replays, RemoveUnexistingManifests and changelog expiration check whether a file is gone. A cache that keeps the metadata of deleted objects answers yes after the file was deleted on OSS, so a recovery could publish a snapshot that references a deleted file. meta now covers file status only; exists uses a target only when io-cache.policy lists exists, so the server vends it only for targets that answer existence as OSS does. The JindoCache mode keeps exists under meta. --- .../paimon/jindo/HadoopCompliantFileIO.java | 8 ++++++-- .../apache/paimon/jindo/IoCacheRouting.java | 20 +++++++++++++++---- .../org/apache/paimon/jindo/JindoFileIO.java | 2 ++ .../paimon/jindo/IoCacheRoutingTest.java | 18 +++++++++++------ .../paimon/jindo/JindoIoCacheRoutingTest.java | 16 +++++++++++++++ 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 195445108927..d365e691efcd 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -72,6 +72,7 @@ public abstract class HadoopCompliantFileIO implements FileIO { protected boolean metaCacheEnabled = false; protected boolean readCacheEnabled = false; protected boolean writeCacheEnabled = false; + protected boolean existsCacheEnabled = false; protected transient volatile Map> fsMap; protected transient volatile Map> jindoCacheFsMap; @@ -120,11 +121,13 @@ public void configure(CatalogContext context) { metaCacheEnabled = cacheRouting.metaCacheEnabled(); readCacheEnabled = cacheRouting.readCacheEnabled(); writeCacheEnabled = cacheRouting.writeCacheEnabled(); + existsCacheEnabled = cacheRouting.existsCacheEnabled(); LOG.info( - "Cache endpoints enabled: meta {}, read {}, write {}, {}", + "Cache endpoints enabled: meta {}, read {}, write {}, exists {}, {}", metaCacheEnabled, readCacheEnabled, writeCacheEnabled, + existsCacheEnabled, cacheRouting); } else { // Keep legacy JindoCache policy matching unchanged; endpoint routing uses exact tokens. @@ -134,6 +137,7 @@ public void configure(CatalogContext context) { metaCacheEnabled = options.get(IO_CACHE_POLICY).contains(META_CACHE_ENABLED_TAG); readCacheEnabled = options.get(IO_CACHE_POLICY).contains(READ_CACHE_ENABLED_TAG); writeCacheEnabled = options.get(IO_CACHE_POLICY).contains(WRITE_CACHE_ENABLED_TAG); + existsCacheEnabled = metaCacheEnabled; String whitelist = options.get(IO_CACHE_WHITELIST_PATH); if (!whitelist.equals("*")) { cacheWhitelistPaths = Lists.newArrayList(whitelist.split(",")); @@ -217,7 +221,7 @@ public FileStatus next() throws IOException { @Override public boolean exists(Path path) throws IOException { org.apache.hadoop.fs.Path hadoopPath = path(path); - boolean shouldCache = metaCacheEnabled && shouldCache(path); + boolean shouldCache = existsCacheEnabled && shouldCache(path); LOG.debug("Exists should cache {} for path {}", shouldCache, path); return getFileSystem(hadoopPath, shouldCache).exists(hadoopPath); } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index 7635d263fe91..6ba00f224a05 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -50,7 +50,7 @@ /** * Selects cache targets for immutable Paimon files using {@code io-cache.targets}, {@code * io-cache.routes} and {@code io-cache.whitelist}. The endpoint mode also carries the normalized - * {@code io-cache.policy} read, metadata and write switches. + * {@code io-cache.policy} read, metadata, existence and write switches. */ final class IoCacheRouting implements Serializable { @@ -59,6 +59,8 @@ final class IoCacheRouting implements Serializable { private static final String META_CACHE_ENABLED_TAG = "meta"; private static final String READ_CACHE_ENABLED_TAG = "read"; private static final String WRITE_CACHE_ENABLED_TAG = "write"; + // exists asks whether a file is still there, which a cache of deleted files can get wrong + private static final String EXISTS_CACHE_ENABLED_TAG = "exists"; private static final String DISABLE_CACHE_TAG = "none"; private static final Pattern TARGET_NAME = Pattern.compile("[a-z][a-z0-9-]*"); @@ -78,6 +80,7 @@ final class IoCacheRouting implements Serializable { private final boolean metaCacheEnabled; private final boolean readCacheEnabled; private final boolean writeCacheEnabled; + private final boolean existsCacheEnabled; private IoCacheRouting( @Nullable String ossEndpoint, @@ -86,7 +89,8 @@ private IoCacheRouting( List dataFilePrefixes, boolean metaCacheEnabled, boolean readCacheEnabled, - boolean writeCacheEnabled) { + boolean writeCacheEnabled, + boolean existsCacheEnabled) { this.ossEndpoint = ossEndpoint; this.targets = targets; this.targetByType = targetByType; @@ -94,6 +98,7 @@ private IoCacheRouting( this.metaCacheEnabled = metaCacheEnabled; this.readCacheEnabled = readCacheEnabled; this.writeCacheEnabled = writeCacheEnabled; + this.existsCacheEnabled = existsCacheEnabled; } /** Creates endpoint routing from the options, or null when routing is not enabled. */ @@ -106,7 +111,9 @@ static IoCacheRouting create(Options options) { boolean metaCache = policy.contains(META_CACHE_ENABLED_TAG); boolean readCache = policy.contains(READ_CACHE_ENABLED_TAG); boolean writeCache = policy.contains(WRITE_CACHE_ENABLED_TAG); - if (policy.contains(DISABLE_CACHE_TAG) || !(metaCache || readCache || writeCache)) { + boolean existsCache = policy.contains(EXISTS_CACHE_ENABLED_TAG); + if (policy.contains(DISABLE_CACHE_TAG) + || !(metaCache || readCache || writeCache || existsCache)) { return null; } // a client-side dlf.oss-endpoint replaces every endpoint the token vends @@ -150,7 +157,8 @@ static IoCacheRouting create(Options options) { dataFilePrefixes(options), metaCache, readCache, - writeCache); + writeCache, + existsCache); } private static Set parsePolicy(@Nullable String value) { @@ -173,6 +181,10 @@ boolean writeCacheEnabled() { return writeCacheEnabled; } + boolean existsCacheEnabled() { + return existsCacheEnabled; + } + /** The OSS endpoint used by requests that do not use a cache target. */ @Nullable String ossEndpoint() { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 4488c3b8aa73..0847a864ec28 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -225,6 +225,8 @@ public Options hadoopOptions(Path path, String opType) { shouldCache = writeCacheEnabled && shouldCache(path); } else if (opType.equalsIgnoreCase("meta")) { shouldCache = metaCacheEnabled && shouldCache(path); + } else if (opType.equalsIgnoreCase("exists")) { + shouldCache = existsCacheEnabled && shouldCache(path); } if (!shouldCache) { return hadoopOptions; diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index bf02eac80f4e..f53595f90ff2 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -63,13 +63,17 @@ public class IoCacheRoutingTest { private static final String ACCEL = "https://accelerator.example.com"; private static final String CLUSTER = "http://cluster.example.com"; private static final String WRITE = "io-cache.policy=meta,read,write"; + private static final String EXISTS = "io-cache.policy=meta,read,exists"; @Test public void testOneCacheTarget() { Options options = single(); assertEndpoint(options, "read", DATA_PATH, CACHE); assertEndpoint(options, "meta", DATA_PATH, CACHE); - assertEndpoint(options, "exists", DATA_PATH, CACHE); + // exists asks whether a file is still there, so it needs its own policy token + assertEndpoint(options, "exists", DATA_PATH, OSS); + assertEndpoint(single(EXISTS), "exists", DATA_PATH, CACHE); + assertEndpoint(single(EXISTS), "exists", SNAPSHOT_PATH, OSS); assertEndpoint(options, "read", MANIFEST_PATH, CACHE); assertEndpoint(options, "read", "manifest/manifest-list-{uuid}-1", CACHE); assertEndpoint(options, "read", "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE); @@ -103,6 +107,9 @@ public void testPolicyAndWhitelist() { Options writeOnly = single("io-cache.policy=write"); assertEndpoint(writeOnly, "write", DATA_PATH, CACHE); assertEndpoint(writeOnly, "exists", DATA_PATH, OSS); + Options existsOnly = single("io-cache.policy=exists"); + assertEndpoint(existsOnly, "exists", DATA_PATH, CACHE); + assertEndpoint(existsOnly, "meta", DATA_PATH, OSS); assertEndpoint(single("-io-cache.whitelist"), "read", INDEX_PATH, CACHE); assertEndpoint(single("io-cache.whitelist=*"), "read", GLOBAL_INDEX_PATH, CACHE); @@ -145,9 +152,10 @@ public void testTwoCacheTargets() { Options options = multi(); assertEndpoint(options, "read", MANIFEST_PATH, ACCEL); assertEndpoint(options, "meta", MANIFEST_PATH, ACCEL); - assertEndpoint(options, "exists", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "exists", MANIFEST_PATH, OSS); + assertEndpoint(multi(EXISTS), "exists", MANIFEST_PATH, ACCEL); assertEndpoint(options, "read", DATA_PATH, CLUSTER); - assertEndpoint(options, "exists", DATA_PATH, CLUSTER); + assertEndpoint(multi(EXISTS), "exists", DATA_PATH, CLUSTER); assertEndpoint(options, "read", INDEX_PATH, CLUSTER); assertEndpoint(options, "read", GLOBAL_INDEX_PATH, CLUSTER); assertEndpoint(options, "read", SNAPSHOT_PATH, OSS); @@ -500,9 +508,7 @@ private static void assertEndpoint(Options options, String op, String path, Stri } JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(copy)); - // exists is a status lookup - String opType = "exists".equals(op) ? "meta" : op; - assertThat(host(fileIO.hadoopOptions(path(path), opType).get("fs.oss.endpoint"))) + assertThat(host(fileIO.hadoopOptions(path(path), op).get("fs.oss.endpoint"))) .as("%s %s with %s", op, path, options.toMap()) .isEqualTo(host(expect)); } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 5d2bec1aa3ea..f0ed08eb3746 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -161,6 +161,21 @@ public void testOnlyReadsAndStatusUseIoCacheRouting() throws IOException { } } + @Test + public void testExistsUsesOssWithoutTheExistsToken() throws IOException { + Options options = cacheTargetOptions(); + options.set("io-cache.policy", "meta,read"); + JindoFileIO fileIO = configuredFileIO(options); + + fileIO.exists(DATA); + fileIO.exists(MANIFEST); + fileIO.getFileStatus(DATA); + verify(ossFs, times(2)).exists(any()); + verify(clusterFs).getFileStatus(any()); + verify(clusterFs, never()).exists(any()); + verify(accelFs, never()).exists(any()); + } + @Test public void testCacheTargetErrorsAreThrown() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); @@ -379,6 +394,7 @@ private static Options cacheTargetOptions() { options.set("io-cache.target.cluster.endpoint", "http://" + CLUSTER_HOST); options.set("io-cache.target.cluster.path-style-access", "true"); options.set("io-cache.routes", "meta=accel;data,bucket-index=cluster"); + options.set("io-cache.policy", "meta,read,exists"); return options; } From 1e6598aafb0a91ea578773c39bb61c44e5f5fd24 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 19:40:40 +0800 Subject: [PATCH 07/15] [jindo] Route metadata and index files only when named as Paimon writes them The broad FileType classifier also matched names such as manifest.parquet, stat-2024.parquet or foo.index, which a Format Table may hold as ordinary files and replace in place, so a routed read could return the old file from a cache. A metadata or index file now uses a target only under the exact name Paimon writes: manifest-, manifest-list-, index-manifest- and stat-{uuid}-{n} (a manifest also with .avro.sidecar), index-{uuid}-{n}, and {kind}-global-index-{uuid}.index. Data files and their file indexes keep the existing name rule. Every other name goes to the origin. --- .../apache/paimon/jindo/IoCacheRouting.java | 33 +++++++++++++++++-- .../paimon/jindo/IoCacheRoutingTest.java | 20 +++++++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 12 ++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index 6ba00f224a05..b17c478c4266 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -67,9 +67,22 @@ final class IoCacheRouting implements Serializable { // Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the // prefix. - private static final Pattern DATA_FILE_SUFFIX = + private static final String UUID = + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; + private static final Pattern DATA_FILE_SUFFIX = Pattern.compile(UUID + "-[0-9]+\\..+"); + + // Other types are routed only under the names Paimon writes; Format Table files may be + // replaced in place. + private static final Pattern META_FILE = Pattern.compile( - "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\\..+"); + "(manifest|manifest-list|index-manifest|stat)-" + + UUID + + "-[0-9]+|manifest-" + + UUID + + "-[0-9]+\\.avro\\.sidecar"); + private static final Pattern BUCKET_INDEX_FILE = Pattern.compile("index-" + UUID + "-[0-9]+"); + private static final Pattern GLOBAL_INDEX_FILE = + Pattern.compile("[a-z0-9_-]+-global-index-" + UUID + "\\.index"); private static final String MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar"; @@ -220,7 +233,21 @@ static FileType cacheableType(Path path, List dataFilePrefixes) { return null; } FileType type = FileType.classify(path); - return type == FileType.DATA ? null : type; + return isPaimonFileName(type, name) ? type : null; + } + + private static boolean isPaimonFileName(FileType type, String name) { + switch (type) { + case META: + return META_FILE.matcher(name).matches(); + case BUCKET_INDEX: + return BUCKET_INDEX_FILE.matcher(name).matches(); + case GLOBAL_INDEX: + return GLOBAL_INDEX_FILE.matcher(name).matches(); + default: + // data files and their file indexes are recognized by isDataFileName + return false; + } } // Manifests, indexes and statistics share the uuid-count shape but have no extension. diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index f53595f90ff2..d4c66e930506 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -87,6 +87,9 @@ public void testOneCacheTarget() { assertEndpoint(options, "read", "dls://bkt/db1.db/t1/" + DATA_PATH, OSS); // the whitelist has no index assertEndpoint(options, "read", INDEX_PATH, OSS); + // a Format Table file named like a manifest may be replaced in place + assertEndpoint(options, "read", "review_external/manifest.parquet", OSS); + assertEndpoint(options, "meta", "review_external/manifest.parquet", OSS); assertEndpoint(options, "write", DATA_PATH, OSS); for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { assertEndpoint(options, op, DATA_PATH, OSS); @@ -255,6 +258,23 @@ public void testCacheableType() { "README")) { assertType(path, null); } + // Format Table files named like metadata or indexes, which may be replaced in place + for (String path : + Arrays.asList( + "dt=1/manifest.parquet", + "dt=1/stat-2024.parquet", + "dt=1/index-a.csv", + "dt=1/foo.index", + "review_external/manifest.parquet", + "manifest/manifest-old", + "manifest/manifest-old.avro.sidecar", + "manifest/manifest-list-{uuid}-1.avro.sidecar", + "statistics/stat-old", + "index/index-old", + "index/my-global-index.index", + "index/btree-global-index-{uuid}-0.index")) { + assertType(path, null); + } assertType("manifest/manifest-{uuid}-0", META); assertType("manifest/manifest-list-{uuid}-1", META); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index f0ed08eb3746..f4b862cb28c0 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -61,7 +61,7 @@ public class JindoIoCacheRoutingTest { private static final String TABLE = "oss://bkt/db1.db/t1"; private static final String UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; private static final Path DATA = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-1.orc"); - private static final Path MANIFEST = new Path(TABLE + "/manifest/manifest-1"); + private static final Path MANIFEST = new Path(TABLE + "/manifest/manifest-" + UUID + "-1"); private static final Path SNAPSHOT = new Path(TABLE + "/snapshot/snapshot-2"); private static final Path LATEST = new Path(TABLE + "/snapshot/LATEST"); private static final String ACCEL_HOST = "cn-hangzhou-j-internal.oss-data-acc.aliyuncs.com"; @@ -331,10 +331,14 @@ Pair createFileSystem( fileIO.configure(CatalogContext.create(cacheTargetOptions())); Pair first = - fileIO.getFileSystemPair(hadoopPath(TABLE + "/manifest/manifest-a"), true); - assertThat(fileIO.getFileSystemPair(hadoopPath(TABLE + "/manifest/manifest-b"), true)) + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/manifest/manifest-" + UUID + "-1"), true); + assertThat( + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/manifest/manifest-" + UUID + "-2"), true)) .isSameAs(first); - fileIO.getFileSystemPair(hadoopPath("oss://other/t/manifest/manifest-c"), true); + fileIO.getFileSystemPair( + hadoopPath("oss://other/t/manifest/manifest-" + UUID + "-3"), true); fileIO.getFileSystemPair( hadoopPath(TABLE + "/dt=1/bucket-0/data-" + UUID + "-3.orc"), true); assertThat(created).containsExactly("accel/bkt", "accel/other", "cluster/bkt"); From bd41c99320acb34aa57103e90651a30db10fefa1 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 16:19:36 +0800 Subject: [PATCH 08/15] [jindo] Route the changelog manifests paimon-rust writes paimon-rust's TableCommit names input-changelog manifests manifest-{uuid}-changelog-{count}, optionally with .avro.sidecar. They are written once under a unique name like every other manifest, so the metadata name rule now accepts the -changelog infix for manifests (not for manifest lists) and a Java reader of such a table routes them the same way as paimon-rust does. --- .../main/java/org/apache/paimon/jindo/IoCacheRouting.java | 8 ++++---- .../java/org/apache/paimon/jindo/IoCacheRoutingTest.java | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index b17c478c4266..c7dad044f7e8 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -71,15 +71,15 @@ final class IoCacheRouting implements Serializable { "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; private static final Pattern DATA_FILE_SUFFIX = Pattern.compile(UUID + "-[0-9]+\\..+"); - // Other types are routed only under the names Paimon writes; Format Table files may be - // replaced in place. + // Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to + // changelog manifests); Format Table files may be replaced in place. private static final Pattern META_FILE = Pattern.compile( - "(manifest|manifest-list|index-manifest|stat)-" + "(manifest-list|index-manifest|stat)-" + UUID + "-[0-9]+|manifest-" + UUID - + "-[0-9]+\\.avro\\.sidecar"); + + "(-changelog)?-[0-9]+(\\.avro\\.sidecar)?"); private static final Pattern BUCKET_INDEX_FILE = Pattern.compile("index-" + UUID + "-[0-9]+"); private static final Pattern GLOBAL_INDEX_FILE = Pattern.compile("[a-z0-9_-]+-global-index-" + UUID + "\\.index"); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index d4c66e930506..fc7cf713b895 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -269,6 +269,8 @@ public void testCacheableType() { "manifest/manifest-old", "manifest/manifest-old.avro.sidecar", "manifest/manifest-list-{uuid}-1.avro.sidecar", + "manifest/manifest-list-{uuid}-changelog-1", + "manifest/manifest-{uuid}-changelog", "statistics/stat-old", "index/index-old", "index/my-global-index.index", @@ -280,6 +282,8 @@ public void testCacheableType() { assertType("manifest/manifest-list-{uuid}-1", META); assertType("manifest/index-manifest-{uuid}-0", META); assertType("manifest/manifest-{uuid}-0.avro.sidecar", META); + assertType("manifest/manifest-{uuid}-changelog-0", META); + assertType("manifest/manifest-{uuid}-changelog-0.avro.sidecar", META); assertType("statistics/stat-{uuid}-0", META); assertType("dt=1/bucket-0/data-{uuid}-0.parquet", DATA); assertType("dt=1/bucket-0/changelog-{uuid}-0.parquet", DATA); From 5af45bcc2197b972bb8825ee24d980cab53841c3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:53:46 +0800 Subject: [PATCH 09/15] [jindo] Give Lance and Vortex the OSS options under io-cache routing Lance and Vortex take hadoopOptions(path, op) and build their own clients, which address the endpoint as virtual-hosted HTTPS and cannot reach an HTTP, IP or path-style io-cache target. With routing, hadoopOptions now returns the OSS options; the routed options stay package-private for JindoFileIO and its tests. JindoCache mode is unchanged. --- .../org/apache/paimon/jindo/JindoFileIO.java | 6 +++ .../paimon/jindo/IoCacheRoutingTest.java | 6 +-- .../paimon/jindo/JindoIoCacheRoutingTest.java | 45 ++++++++++++------- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 0847a864ec28..0be4a335c7b3 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -218,6 +218,12 @@ static Options withEndpoint(Options base, @Nullable String endpoint) { */ @Override public Options hadoopOptions(Path path, String opType) { + // Lance and Vortex build their own clients from these options; they only reach the origin. + return cacheRouting != null ? hadoopOptions : routedHadoopOptions(path, opType); + } + + /** The options of the endpoint that a request of this type on this path uses. */ + Options routedHadoopOptions(Path path, String opType) { boolean shouldCache = false; if (opType.equalsIgnoreCase("read")) { shouldCache = readCacheEnabled && shouldCache(path); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index fc7cf713b895..ab402ab6c536 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -361,9 +361,9 @@ public void testCachePolicyUsesCompleteCaseInsensitiveTokens( options.set("io-cache.policy", policy); JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(options)); - assertThat(host(fileIO.hadoopOptions(DATA_FILE, "read").get("fs.oss.endpoint"))) + assertThat(host(fileIO.routedHadoopOptions(DATA_FILE, "read").get("fs.oss.endpoint"))) .isEqualTo(readCache ? "cache:8080" : "oss-cn-hangzhou.aliyuncs.com"); - assertThat(host(fileIO.hadoopOptions(DATA_FILE, "meta").get("fs.oss.endpoint"))) + assertThat(host(fileIO.routedHadoopOptions(DATA_FILE, "meta").get("fs.oss.endpoint"))) .isEqualTo( metaCache ? "cache:8080" @@ -532,7 +532,7 @@ private static void assertEndpoint(Options options, String op, String path, Stri } JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(copy)); - assertThat(host(fileIO.hadoopOptions(path(path), op).get("fs.oss.endpoint"))) + assertThat(host(fileIO.routedHadoopOptions(path(path), op).get("fs.oss.endpoint"))) .as("%s %s with %s", op, path, options.toMap()) .isEqualTo(host(expect)); } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index f4b862cb28c0..7d972d634701 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -85,31 +85,45 @@ public void testOptionsOfEachCacheTarget() { JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(cacheTargetOptions())); - Options cluster = fileIO.hadoopOptions(DATA, "read"); + Options cluster = fileIO.routedHadoopOptions(DATA, "read"); assertThat(cluster.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); assertThat(cluster.get("fs.oss.https.enable")).isEqualTo("false"); assertThat(cluster.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); assertThat(cluster.get("fs.oss.region")).isEqualTo("cn-hangzhou"); assertThat(cluster.get(DLF_CACHE_KEY)).isEqualTo("true"); assertThat(cluster.containsKey("fs.xengine")).isFalse(); - assertThat(fileIO.hadoopOptions(DATA, "meta")).isSameAs(cluster); + assertThat(fileIO.routedHadoopOptions(DATA, "meta")).isSameAs(cluster); - Options accel = fileIO.hadoopOptions(MANIFEST, "meta"); + Options accel = fileIO.routedHadoopOptions(MANIFEST, "meta"); assertThat(accel.get("fs.oss.endpoint")).isEqualTo(ACCEL_HOST); assertThat(accel.get("fs.oss.https.enable")).isEqualTo("true"); assertThat(accel.get("fs.oss.second.level.domain.enable")).isEqualTo("false"); assertThat(accel.get("fs.oss.region")).isEqualTo("cn-shanghai"); - Options oss = fileIO.hadoopOptions(DATA, "write"); + Options oss = fileIO.routedHadoopOptions(DATA, "write"); assertThat(oss.get("fs.oss.endpoint")).isEqualTo(OSS_HOST); assertThat(oss.get("fs.oss.https.enable")).isEqualTo("true"); assertThat(oss.get("fs.oss.region")).isEqualTo("cn-hangzhou"); assertThat(oss.containsKey(DLF_CACHE_KEY)).isFalse(); for (Path path : new Path[] {SNAPSHOT, LATEST, new Path(TABLE + "/dt=1/bucket-0/000000_0")}) { - assertThat(fileIO.hadoopOptions(path, "read")).as(path.toString()).isSameAs(oss); + assertThat(fileIO.routedHadoopOptions(path, "read")).as(path.toString()).isSameAs(oss); + } + assertThat(fileIO.routedHadoopOptions(DATA, "list")).isSameAs(oss); + } + + @Test + public void testLanceAndVortexGetTheOssOptions() { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + // their converters build an https virtual-hosted address, which only the origin serves + Options oss = fileIO.routedHadoopOptions(DATA, "write"); + for (String op : new String[] {"read", "write", "meta"}) { + for (Path path : new Path[] {DATA, MANIFEST, LATEST}) { + assertThat(fileIO.hadoopOptions(path, op)).as(path + " " + op).isSameAs(oss); + } } - assertThat(fileIO.hadoopOptions(DATA, "list")).isSameAs(oss); } @Test @@ -221,7 +235,7 @@ public void testWritesFollowTheRouteWithWritePolicy() throws IOException { fileIO.newOutputStream(MANIFEST, false); fileIO.newOutputStream(SNAPSHOT, false); fileIO.newTwoPhaseOutputStream(next, false); - assertThat(fileIO.hadoopOptions(DATA, "write").get("fs.oss.endpoint")) + assertThat(fileIO.routedHadoopOptions(DATA, "write").get("fs.oss.endpoint")) .isEqualTo(CLUSTER_HOST); verify(clusterFs).create(any(), anyBoolean()); verify(accelFs).create(any(), anyBoolean()); @@ -248,11 +262,12 @@ public void testSingleCacheTarget() { JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(options)); - Options read = fileIO.hadoopOptions(DATA, "read"); + Options read = fileIO.routedHadoopOptions(DATA, "read"); assertThat(read.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); assertThat(read.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); - assertThat(fileIO.hadoopOptions(MANIFEST, "meta")).isSameAs(read); - assertThat(fileIO.hadoopOptions(DATA, "write").get("fs.oss.endpoint")).isEqualTo(OSS_HOST); + assertThat(fileIO.routedHadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.routedHadoopOptions(DATA, "write").get("fs.oss.endpoint")) + .isEqualTo(OSS_HOST); } @Test @@ -263,14 +278,14 @@ public void testJindoCacheTakesPrecedence() { fileIO.configure(CatalogContext.create(options)); assertThat(fileIO.cacheRouting).isNull(); - Options read = fileIO.hadoopOptions(DATA, "read"); + Options read = fileIO.routedHadoopOptions(DATA, "read"); assertThat(read.get("fs.xengine")).isEqualTo("jindocache"); assertThat(read.get("fs.oss.endpoint")).isEqualTo("http://" + CLUSTER_HOST); - Options write = fileIO.hadoopOptions(DATA, "write"); + Options write = fileIO.routedHadoopOptions(DATA, "write"); assertThat(write.containsKey("fs.xengine")).isFalse(); // legacy whitelist-path matching: "manifest" is in the default whitelist - assertThat(fileIO.hadoopOptions(MANIFEST, "meta")).isSameAs(read); - assertThat(fileIO.hadoopOptions(LATEST, "read")).isSameAs(write); + assertThat(fileIO.routedHadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.routedHadoopOptions(LATEST, "read")).isSameAs(write); } @Test @@ -288,7 +303,7 @@ public void testWithoutIoCacheRouting() { fileIO.configure(CatalogContext.create(options)); assertThat(fileIO.cacheRouting).isNull(); for (String op : new String[] {"read", "meta", "write"}) { - Options hadoopOptions = fileIO.hadoopOptions(DATA, op); + Options hadoopOptions = fileIO.routedHadoopOptions(DATA, op); assertThat(hadoopOptions.get("fs.oss.endpoint")) .isEqualTo(options.get("fs.oss.endpoint")); assertThat(hadoopOptions.containsKey("fs.oss.https.enable")).isFalse(); From a66792f397914ba2225869aaa4cd98e744683a47 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 09:34:32 +0800 Subject: [PATCH 10/15] [jindo] Recognize postpone and paimon-rust managed BLOB file names in io-cache routing The postpone writer names its files {prefix}-u-{commit user}-s-{write id}-w-{uuid}-{count}.{extension}, which did not match {prefix}{uuid}-{count}.{extension}, so their reads and writes always used the origin. paimon-rust names its managed BLOB packs {prefix}{uuid}.managed.blob, without the count this client writes. Both are written once under a unique name and are now data files, with the .index and .blobref files named after them, so all three clients route the same names (apache/paimon-rust#1014). --- .../apache/paimon/jindo/IoCacheRouting.java | 8 ++++--- .../paimon/jindo/IoCacheRoutingTest.java | 23 +++++++++++++++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 11 +++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index c7dad044f7e8..0576211f2075 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -65,11 +65,13 @@ final class IoCacheRouting implements Serializable { private static final Pattern TARGET_NAME = Pattern.compile("[a-z][a-z0-9-]*"); - // Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the - // prefix. + // What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone + // writer's -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack {uuid}.managed.blob. private static final String UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; - private static final Pattern DATA_FILE_SUFFIX = Pattern.compile(UUID + "-[0-9]+\\..+"); + private static final Pattern DATA_FILE_SUFFIX = + Pattern.compile( + "(?:-u-.+-s-[0-9]+-w-)?" + UUID + "-[0-9]+\\..+|" + UUID + "\\.managed\\.blob"); // Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to // changelog manifests); Format Table files may be replaced in place. diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index ab402ab6c536..2dd9dfab9ce0 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -255,6 +255,12 @@ public void testCacheableType() { "dt=1/part-00000-1a2b.snappy.parquet", "dt=1/bucket-0/data-1.parquet", "dt=1/bucket-0/custom-{uuid}-0.orc", + "bucket-postpone/data--u-user-s-x-w-{uuid}-0.parquet", + "bucket-postpone/data--u--s-1-w-{uuid}-0.parquet", + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.parquet", + "dt=1/bucket-0/data-{uuid}.blob", + "dt=1/bucket-0/data-{uuid}.managed.blob.{uuid}.tmp", + "dt=1/bucket-0/data-old.managed.blob", "README")) { assertType(path, null); } @@ -290,6 +296,19 @@ public void testCacheableType() { assertType("dt=1/bucket-0/data-{uuid}-0.blob", DATA); assertType("dt=1/8b1f7c2e/data-{uuid}-0.parquet", DATA); assertType("dt=1/bucket-0/data-{uuid}-0.parquet.index", FILE_INDEX); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.blobref", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.managed.blob", DATA); + assertType("dt=1/bucket-0/data-{uuid}.managed.blob", DATA); + // the postpone writer puts its commit user and write id before {uuid}-{count} + for (String postpone : + Arrays.asList( + "bucket-postpone/data--u-io-cache-user-s-7-w-{uuid}-0.parquet", + "dt=1/bucket-postpone/data--u-{uuid}-s-2147483647-w-{uuid}-12.orc", + "bucket-postpone/changelog--u-user-s-0-w-{uuid}-0.parquet")) { + assertType(postpone, DATA); + assertType(postpone + ".index", FILE_INDEX); + assertType(postpone + ".blobref", DATA); + } assertType("index/index-{uuid}-0", BUCKET_INDEX); assertType("index/btree-global-index-{uuid}.index", GLOBAL_INDEX); assertType("index/lumina-global-index-{uuid}.index", GLOBAL_INDEX); @@ -298,6 +317,10 @@ public void testCacheableType() { @Test public void testCacheableTypeWithFilePrefixes() { assertType("dt=1/bucket-0/custom-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); + assertType( + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.orc", + DATA, + "data-file.prefix=custom-"); assertType("dt=1/bucket-0/data-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); assertType("dt=1/bucket-0/cl-{uuid}-0.orc", DATA, "changelog-file.prefix=cl-"); assertType("dt=1/bucket-0/ unknown.parquet", null, "data-file.prefix= "); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 7d972d634701..3426b9f85d98 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -254,6 +254,17 @@ public void testWritesFollowTheRouteWithWritePolicy() throws IOException { verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testPostponeFilesAndBlobPacksUseTheirTarget() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + String postpone = "/bucket-postpone/data--u-user-s-7-w-" + UUID + "-0.avro"; + fileIO.newInputStream(new Path(TABLE + postpone)).close(); + fileIO.newInputStream(new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + ".managed.blob")) + .close(); + verify(clusterFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + @Test public void testSingleCacheTarget() { Options options = baseOptions(); From 3f51a2de3e8c9c15f570e44b7d74dd7cf8dad4d3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:44:55 +0800 Subject: [PATCH 11/15] [core] Give each table its own io-cache view of its FileIO io-cache routing chose the endpoint of a request from the file name alone, for every table a FileIO served. Review of apache/paimon-rust#1014 found that a name shows neither who owns a file nor which prefixes a table has used, and this client has the same gaps: - Format, Object, Iceberg and Lance tables hold files that other writers may replace in place under the same name. Their FileIO is now origin only. - A row may point at any file. A blob-descriptor-field value, or a BLOB written from a descriptor, names a file outside the table that its writer may replace, even when it is named like a data file. A FileStoreTable now gives io-cache routing its location, and only files under it may use a cache target; files under data-file.external-paths are read from origin, as without routing. Descriptor readers created from the catalog context serve no table, so their FileIO is origin only. - Data files named with the data-file.prefix or changelog-file.prefix of the schema, of a copy's options or of an older schema were not recognized. The view passes the prefixes of the table's schema, and the router lists the branch's older schemas on OSS, once, the first time a file under the table is named like a data file with none of the known prefixes. FileIO gets two default no-op methods, originOnly() and withTableView(). AbstractFileStoreTable's constructor applies the view, so every copy, time travel and branch takes the one of its own schema. RESTTokenFileIO, ResolvingFileIO and CachingFileIO apply the view to the FileIO they wrap; a JindoFileIO view shares the file systems and the presigner of its base and closes nothing. IoCacheTableRoutingTest writes and reads through a FileSystemCatalog over a FileIO whose target keeps the first copy it served while origin holds a new one, as the paimon-rust review does: descriptor fields of primary-key and append tables, a BLOB copied from a descriptor, data files under an external path, a table read after ALTER of its prefix, and a branch with its own prefix. Each case but the branch one fails without its part of this change; the jindo tests cover the router. --- .../java/org/apache/paimon/fs/FileIO.java | 10 + .../apache/paimon/fs/FilePrefixHistory.java | 30 ++ .../org/apache/paimon/fs/ResolvingFileIO.java | 31 ++ .../org/apache/paimon/fs/TableFileIOView.java | 190 +++++++++ .../apache/paimon/fs/cache/CachingFileIO.java | 23 ++ .../apache/paimon/rest/RESTTokenFileIO.java | 55 ++- .../apache/paimon/utils/UriReaderFactory.java | 3 +- .../apache/paimon/fs/ResolvingFileIOTest.java | 30 ++ .../paimon/fs/cache/CachingFileIOTest.java | 29 ++ .../paimon/rest/RESTTokenFileIOTest.java | 48 +++ .../paimon/utils/UriReaderFactoryTest.java | 25 ++ .../paimon/table/AbstractFileStoreTable.java | 3 +- .../org/apache/paimon/table/FormatTable.java | 3 +- .../apache/paimon/table/TableFileIOViews.java | 99 +++++ .../paimon/table/iceberg/IcebergTable.java | 10 +- .../apache/paimon/table/lance/LanceTable.java | 9 +- .../paimon/table/object/ObjectTable.java | 8 +- .../paimon/table/IoCacheTableRoutingTest.java | 368 ++++++++++++++++++ .../paimon/table/TableFileIOViewTest.java | 213 ++++++++++ .../paimon/jindo/HadoopCompliantFileIO.java | 15 +- .../apache/paimon/jindo/IoCacheRouting.java | 148 ++++++- .../org/apache/paimon/jindo/JindoFileIO.java | 56 ++- .../paimon/jindo/IoCacheRoutingTest.java | 148 +++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 130 +++++++ 24 files changed, 1666 insertions(+), 18 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index 16efe5f6a627..af47af570c91 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -80,6 +80,16 @@ public interface FileIO extends Serializable, Closeable { /** Set filesystem options at runtime. Usually used for job-level settings. */ default void setRuntimeContext(Map options) {} + /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ + default FileIO originOnly() { + return this; + } + + /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ + default FileIO withTableView(TableFileIOView view) { + return this; + } + /** * Opens an SeekableInputStream at the indicated Path. * diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java b/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java new file mode 100644 index 000000000000..1cc50d02f3e7 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +/** Loads the data file prefixes of the schemas a table has had, for io-cache routing. */ +public interface FilePrefixHistory extends Serializable { + + /** Reads them through {@code fileIO}, which sends no request to an io-cache target. */ + List filePrefixes(FileIO fileIO) throws IOException; +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java index 5091acaaff59..4f8cd210dcf0 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java @@ -23,6 +23,8 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.Serializable; import java.time.Duration; @@ -44,6 +46,31 @@ public class ResolvingFileIO implements FileIO { private CatalogContext context; + // a table view resolves through its base and applies the view to each FileIO + @Nullable private ResolvingFileIO base; + private TableFileIOView view = TableFileIOView.NONE; + + @Override + public FileIO originOnly() { + return withView(view.originOnly()); + } + + @Override + public FileIO withTableView(TableFileIOView tableView) { + return withView(view.merge(tableView)); + } + + private FileIO withView(TableFileIOView newView) { + if (newView == view) { + return this; + } + ResolvingFileIO result = new ResolvingFileIO(); + result.context = context; + result.base = base == null ? this : base; + result.view = newView; + return result; + } + // TODO, how to decide the real fileio is object store or not? @Override public boolean isObjectStore() { @@ -143,6 +170,10 @@ public String createBlobPresignedUrl( public FileIO fileIO(Path path) throws IOException { CacheKey cacheKey = new CacheKey(path.toUri().getScheme(), path.toUri().getAuthority()); + if (base != null) { + FileIO resolved = base.fileIO(path); + return fileIOMap.computeIfAbsent(cacheKey, k -> view.apply(resolved)); + } return fileIOMap.computeIfAbsent( cacheKey, k -> { diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java b/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java new file mode 100644 index 000000000000..7f14eeae6fc5 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * What io-cache routing knows about the table a FileIO view serves: the locations it owns, the + * prefixes of its data files and where to find the prefixes of its older schemas. + */ +public final class TableFileIOView implements Serializable { + + private static final long serialVersionUID = 2L; + + public static final TableFileIOView NONE = + new TableFileIOView(false, Collections.emptyList(), Collections.emptyList(), null); + + private static final TableFileIOView ORIGIN_ONLY = + new TableFileIOView(true, Collections.emptyList(), Collections.emptyList(), null); + + private final boolean originOnly; + private final List roots; + private final List filePrefixes; + @Nullable private final FilePrefixHistory history; + + private TableFileIOView( + boolean originOnly, + List roots, + List filePrefixes, + @Nullable FilePrefixHistory history) { + this.originOnly = originOnly; + this.roots = roots; + this.filePrefixes = filePrefixes; + this.history = history; + } + + /** The view of a table whose files are under {@code roots}. */ + public static TableFileIOView of( + Collection roots, + Collection filePrefixes, + @Nullable FilePrefixHistory history) { + List normalizedRoots = new ArrayList<>(); + for (Path root : roots) { + addIfAbsent(normalizedRoots, normalizeRoot(root)); + } + List prefixes = new ArrayList<>(); + for (String prefix : filePrefixes) { + if (prefix != null && !prefix.trim().isEmpty()) { + addIfAbsent(prefixes, prefix); + } + } + return new TableFileIOView( + false, + Collections.unmodifiableList(normalizedRoots), + Collections.unmodifiableList(prefixes), + history); + } + + /** See {@link FileIO#originOnly()}. */ + public TableFileIOView originOnly() { + return ORIGIN_ONLY; + } + + /** This view also serving {@code other}'s table, with its history; origin only stays so. */ + public TableFileIOView merge(TableFileIOView other) { + if (originOnly || other.isNone()) { + return this; + } + if (other.originOnly || isNone()) { + return other; + } + List mergedRoots = new ArrayList<>(roots); + other.roots.forEach(root -> addIfAbsent(mergedRoots, root)); + List mergedPrefixes = new ArrayList<>(filePrefixes); + other.filePrefixes.forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); + TableFileIOView merged = + new TableFileIOView( + false, + Collections.unmodifiableList(mergedRoots), + Collections.unmodifiableList(mergedPrefixes), + other.history == null ? history : other.history); + return merged.equals(this) ? this : merged; + } + + public boolean isOriginOnly() { + return originOnly; + } + + public boolean isNone() { + return this == NONE || (!originOnly && roots.isEmpty() && filePrefixes.isEmpty()); + } + + /** Normalized {@code scheme://authority/path} of each location the table owns. */ + public List roots() { + return roots; + } + + public List filePrefixes() { + return filePrefixes; + } + + @Nullable + public FilePrefixHistory history() { + return history; + } + + /** Whether {@code path} is one of the roots or under one. */ + public static boolean isUnder(List roots, Path path) { + String location = normalizeRoot(path); + for (String root : roots) { + if (location.equals(root) || location.startsWith(root + "/")) { + return true; + } + } + return false; + } + + public FileIO apply(FileIO fileIO) { + if (originOnly) { + return fileIO.originOnly(); + } + return isNone() ? fileIO : fileIO.withTableView(this); + } + + private static String normalizeRoot(Path path) { + URI uri = path.toUri(); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + String authority = uri.getAuthority() == null ? "" : uri.getAuthority(); + String location = uri.getPath() == null ? "" : uri.getPath(); + while (location.endsWith("/")) { + location = location.substring(0, location.length() - 1); + } + return scheme + "://" + authority + location; + } + + private static void addIfAbsent(List list, String value) { + if (!list.contains(value)) { + list.add(value); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof TableFileIOView)) { + return false; + } + TableFileIOView that = (TableFileIOView) o; + return originOnly == that.originOnly + && roots.equals(that.roots) + && filePrefixes.equals(that.filePrefixes) + && Objects.equals(history, that.history); + } + + @Override + public int hashCode() { + return Objects.hash(originOnly, roots, filePrefixes, history); + } + + @Override + public String toString() { + return originOnly + ? "TableFileIOView{originOnly}" + : "TableFileIOView{roots=" + roots + ", prefixes=" + filePrefixes + "}"; + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java index d943abcfbcf0..0e3e49afcad4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -216,6 +217,28 @@ public void setRuntimeContext(Map options) { delegate.setRuntimeContext(options); } + @Override + public FileIO originOnly() { + return withDelegate(delegate.originOnly()); + } + + @Override + public FileIO withTableView(TableFileIOView view) { + return withDelegate(delegate.withTableView(view)); + } + + // a view shares this wrapper's cache + private FileIO withDelegate(FileIO view) { + return view == delegate + ? this + : new CachingFileIO( + view, + getOrCreateCacheManager(), + whitelist, + cacheConfiguration, + cacheNamespace); + } + @Override public void close() throws IOException { SharedCacheManager shared; diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index 830fe058c43c..1690b8027132 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -27,12 +27,14 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; import org.apache.paimon.options.Options; import org.apache.paimon.rest.responses.GetTableTokenResponse; import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ThreadUtils; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; @@ -44,6 +46,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.time.Duration; import java.util.Map; @@ -112,12 +116,57 @@ static long fileIOCacheMaximumSize() { // Server again after serialization private volatile RESTToken token; + // applied to the FileIO of each token + private final TableFileIOView view; + private transient volatile Pair lastView; + public RESTTokenFileIO( CatalogContext catalogContext, RESTApi apiInstance, Identifier identifier, Path path) { + this(catalogContext, apiInstance, identifier, path, TableFileIOView.NONE, null); + } + + private RESTTokenFileIO( + CatalogContext catalogContext, + RESTApi apiInstance, + Identifier identifier, + Path path, + TableFileIOView view, + @Nullable RESTToken token) { this.catalogContext = catalogContext; this.apiInstance = apiInstance; this.identifier = identifier; this.path = path; + this.view = view; + this.token = token; + } + + @Override + public FileIO originOnly() { + return withView(view.originOnly()); + } + + @Override + public FileIO withTableView(TableFileIOView tableView) { + return withView(view.merge(tableView)); + } + + private FileIO withView(TableFileIOView newView) { + return newView == view + ? this + : new RESTTokenFileIO( + catalogContext, apiInstance, identifier, path, newView, token); + } + + private FileIO viewOf(FileIO fileIO) { + if (view.isNone()) { + return fileIO; + } + Pair last = lastView; + if (last == null || last.getLeft() != fileIO) { + last = Pair.of(fileIO, view.apply(fileIO)); + lastView = last; + } + return last.getRight(); } @Override @@ -247,13 +296,13 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc FileIO fileIO = FILE_IO_CACHE.getIfPresent(currentToken); if (fileIO != null) { - return new FileIOWithToken(fileIO, currentToken); + return new FileIOWithToken(viewOf(fileIO), currentToken); } synchronized (FILE_IO_CACHE) { fileIO = FILE_IO_CACHE.getIfPresent(currentToken); if (fileIO != null) { - return new FileIOWithToken(fileIO, currentToken); + return new FileIOWithToken(viewOf(fileIO), currentToken); } Options options = catalogContext.options(); @@ -267,7 +316,7 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc catalogContext.fallbackIO()); fileIO = FileIO.get(path, context); FILE_IO_CACHE.put(currentToken, fileIO); - return new FileIOWithToken(fileIO, currentToken); + return new FileIOWithToken(viewOf(fileIO), currentToken); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java index 4d1f84cfcbb3..866d9f7bf467 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java @@ -96,7 +96,8 @@ protected UriReader newReader(URI uri) { try { FileIO createdFileIO = FileIO.get(new Path(uri), Objects.requireNonNull(context)); - return UriReader.fromFile(createdFileIO); + // no table owns what a descriptor points at here, and its writer may replace it + return UriReader.fromFile(createdFileIO.originOnly()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java index 6330d68daafd..0f63bf58368e 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java @@ -39,6 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -57,6 +58,35 @@ public void setUp() { resolvingFileIO.configure(catalogContext); } + @Test + public void testTableViewsApplyToEachResolvedFileIO() throws IOException { + FileIO oss = mock(FileIO.class); + FileIO originOnly = mock(FileIO.class); + when(oss.originOnly()).thenReturn(originOnly); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.getScheme()).thenReturn("oss"); + when(loader.load(any())).thenReturn(oss); + ResolvingFileIO fileIO = new ResolvingFileIO(); + fileIO.configure(CatalogContext.create(new Options(), loader, null)); + Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); + + FileIO view = fileIO.originOnly(); + assertInstanceOf(ResolvingFileIO.class, view); + assertSame(view, view.originOnly()); + assertSame(originOnly, ((ResolvingFileIO) view).fileIO(data)); + assertSame(oss, fileIO.fileIO(data)); + // loading a FileIO checks access with exists + clearInvocations(oss); + view.exists(data); + verify(originOnly).exists(data); + verify(oss, never()).exists(any()); + // a FileIO without io-cache routing is its own view + Path local = new Path("file:///tmp/x"); + assertSame( + resolvingFileIO.fileIO(local), + ((ResolvingFileIO) resolvingFileIO.originOnly()).fileIO(local)); + } + @Test public void testFileIONullSchemeReturnsFallbackFileIO() throws IOException { Path path = new Path("/path/to/file"); diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index b4f3c46373dd..84aabb5d7c76 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.VectoredReadUtils; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -43,6 +44,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -216,6 +218,33 @@ private CachingFileIO newCachingFileIO( return new CachingFileIO(delegate, cache, whitelist); } + @Test + void testTableViewsWrapTheViewOfTheDelegate() throws IOException { + FileIO delegate = mock(FileIO.class); + FileIO originOnly = mock(FileIO.class); + when(delegate.originOnly()).thenReturn(originOnly); + when(delegate.withTableView(any())).thenReturn(delegate); + CachingFileIO cachingIO = + new CachingFileIO( + delegate, + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META)); + Path data = new Path("oss://bucket/t/bucket-0/data-1.orc"); + + FileIO view = cachingIO.originOnly(); + assertThat(view).isInstanceOf(CachingFileIO.class).isNotSameAs(cachingIO); + view.exists(data); + verify(originOnly).exists(data); + verify(delegate, never()).exists(any()); + // a delegate without anything to narrow keeps the wrapper + TableFileIOView tableView = + TableFileIOView.of( + Collections.singletonList(new Path("oss://bucket/t")), + Collections.singletonList("custom-"), + null); + assertThat(cachingIO.withTableView(tableView)).isSameAs(cachingIO); + } + @Test void testShortRemoteReadIsNotCachedAsZeroPaddedBlock() throws IOException { byte[] data = "truncated".getBytes(); diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java index e80f097fcf2e..ac2ca9ff7e73 100644 --- a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.RemoteIterator; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.options.Options; import org.apache.paimon.rest.responses.GetTableTokenResponse; @@ -256,6 +257,53 @@ void testFileIOCreationFailureSurfacesAsCheckedIOException() throws IOException assertThatThrownBy(() -> fileIO.exists(tableRoot)).isInstanceOf(IOException.class); } + @Test + void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { + Path tableRoot = new Path("oss://bucket/table"); + FileIO delegate = mock(FileIO.class); + FileIO originOnly = mock(FileIO.class); + FileIO prefixed = mock(FileIO.class); + TableFileIOView tableView = + TableFileIOView.of( + Collections.singletonList(tableRoot), + Collections.singletonList("custom-"), + null); + when(delegate.originOnly()).thenReturn(originOnly); + when(delegate.withTableView(tableView)).thenReturn(prefixed); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenReturn(delegate); + when(loader.getScheme()).thenReturn("oss"); + RESTApi api = mock(RESTApi.class); + Identifier identifier = Identifier.create("db", "table"); + when(api.loadTableToken(identifier)) + .thenReturn( + new GetTableTokenResponse( + Collections.singletonMap("token", UUID.randomUUID().toString()), + Long.MAX_VALUE)); + RESTTokenFileIO fileIO = + new RESTTokenFileIO( + CatalogContext.create(new Options(), loader, null), + api, + identifier, + tableRoot); + + FileIO prefixView = fileIO.withTableView(tableView); + assertThat(prefixView).isInstanceOf(RESTTokenFileIO.class).isNotSameAs(fileIO); + assertThat(prefixView.withTableView(tableView)).isSameAs(prefixView); + assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); + assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); + // the view of the token's FileIO is built once + verify(delegate, times(1)).withTableView(any()); + + FileIO originView = fileIO.originOnly(); + assertThat(originView.originOnly()).isSameAs(originView); + Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); + originView.exists(data); + verify(originOnly).exists(data); + verify(delegate, never()).exists(data); + assertThat(fileIO.fileIO()).isSameAs(delegate); + } + @Test void testTryToWriteAtomicReachesInnerOverride() throws IOException { Path tableRoot = new Path("oss://bucket/table"); diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java index 16ea5676fb42..94992505bc91 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java @@ -19,6 +19,8 @@ package org.apache.paimon.utils; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.IsolatedDirectoryFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; @@ -40,6 +42,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Test for {@link UriReaderFactory}. */ public class UriReaderFactoryTest { @@ -66,6 +73,24 @@ public void tearDownHttpServer() { } } + @Test + public void testFilesOfDescriptorsAreReadFromTheOriginOfTheCreatedFileIO() throws Exception { + // a descriptor may name a file another writer replaces, and no table owns it here + FileIO created = mock(FileIO.class); + FileIO originOnly = mock(FileIO.class); + when(created.originOnly()).thenReturn(originOnly); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.getScheme()).thenReturn("oss"); + when(loader.load(any())).thenReturn(created); + UriReaderFactory ossFactory = + new UriReaderFactory(CatalogContext.create(new Options(), loader, null)); + String uri = "oss://bkt/import/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1.png"; + + ossFactory.create(uri).newInputStream(uri); + verify(originOnly).newInputStream(new Path(uri)); + verify(created, never()).newInputStream(any()); + } + @Test public void testCreateHttpUriReader() { UriReader reader = factory.create("http://example.com/file.txt"); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 5a04aeb116c1..15e0d3fa7a60 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -117,7 +117,8 @@ protected AbstractFileStoreTable( Path path, TableSchema tableSchema, CatalogEnvironment catalogEnvironment) { - this.fileIO = fileIO; + // every copy, branch and time travel comes through here with its own schema + this.fileIO = TableFileIOViews.forTable(fileIO, path, tableSchema.options()); this.path = path; if (!tableSchema.options().containsKey(PATH.key())) { // make sure table is always available diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 7f3ee12d6ef2..8c8b03bad950 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -186,8 +186,9 @@ public Builder partitionManager(@Nullable FormatTablePartitionManager partitionM } public FormatTable build() { + // other writers may replace Format Table files in place return new FormatTableImpl( - fileIO, + fileIO == null ? null : fileIO.originOnly(), identifier, rowType, partitionKeys, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java new file mode 100644 index 000000000000..7411ac5a071a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FilePrefixHistory; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.utils.StringUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.paimon.CoreOptions.CHANGELOG_FILE_PREFIX; +import static org.apache.paimon.CoreOptions.DATA_FILE_PREFIX; + +/** Builds the {@link TableFileIOView} every {@link FileStoreTable} applies to its FileIO. */ +final class TableFileIOViews { + + private TableFileIOViews() {} + + /** The FileIO of the table at {@code path} with {@code options}. */ + static FileIO forTable(FileIO fileIO, Path path, Map options) { + List prefixes = new ArrayList<>(2); + prefixes.add(options.get(DATA_FILE_PREFIX.key())); + prefixes.add(options.get(CHANGELOG_FILE_PREFIX.key())); + // only files under the table's location use io-cache, not those under external paths + return fileIO.withTableView( + TableFileIOView.of( + Collections.singletonList(path), + prefixes, + new SchemaFilePrefixHistory(path, CoreOptions.branch(options)))); + } + + /** The data and changelog file prefixes of every schema of a branch, read from storage. */ + static final class SchemaFilePrefixHistory implements FilePrefixHistory { + + private static final long serialVersionUID = 1L; + + private final Path path; + private final String branch; + + SchemaFilePrefixHistory(Path path, String branch) { + this.path = path; + this.branch = branch; + } + + @Override + public List filePrefixes(FileIO fileIO) { + List prefixes = new ArrayList<>(); + for (TableSchema schema : new FileSystemSchemaManager(fileIO, path, branch).listAll()) { + for (String key : + new String[] {DATA_FILE_PREFIX.key(), CHANGELOG_FILE_PREFIX.key()}) { + String prefix = schema.options().get(key); + if (!StringUtils.isNullOrWhitespaceOnly(prefix) && !prefixes.contains(prefix)) { + prefixes.add(prefix); + } + } + } + return prefixes; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SchemaFilePrefixHistory)) { + return false; + } + SchemaFilePrefixHistory that = (SchemaFilePrefixHistory) o; + return path.equals(that.path) && branch.equals(that.branch); + } + + @Override + public int hashCode() { + return Objects.hash(path, branch); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java index f8c67fc77681..1df00a20dc89 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java @@ -93,8 +93,16 @@ public Builder uuid(String uuid) { } public IcebergTable build() { + // other engines write Iceberg tables, so their files never use io-cache return new IcebergTableImpl( - identifier, fileIO, rowType, partitionKeys, location, options, comment, uuid); + identifier, + fileIO == null ? null : fileIO.originOnly(), + rowType, + partitionKeys, + location, + options, + comment, + uuid); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java index 93146a41a814..0c5a2f9cd63a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java @@ -82,7 +82,14 @@ public Builder comment(String comment) { } public LanceTable build() { - return new LanceTableImpl(identifier, fileIO, rowType, location, options, comment); + // other engines write Lance datasets, so their files never use io-cache + return new LanceTableImpl( + identifier, + fileIO == null ? null : fileIO.originOnly(), + rowType, + location, + options, + comment); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java index 98c95d6d8ed8..6b22232d31a4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java @@ -91,7 +91,13 @@ public ObjectTable.Builder comment(String comment) { } public ObjectTable build() { - return new ObjectTableImpl(identifier, fileIO, location, options, comment); + // other writers manage the objects, so they may change under the same name + return new ObjectTableImpl( + identifier, + fileIO == null ? null : fileIO.originOnly(), + location, + options, + comment); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java new file mode 100644 index 000000000000..fa1b06b6ef59 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.UriReader; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Reads through an io-cache target that keeps the first copy it served while origin moves on, as + * the review of apache/paimon-rust#1014 does, through the catalog API and real writers. + */ +class IoCacheTableRoutingTest { + + private static final byte[] OLD = "OLD".getBytes(StandardCharsets.UTF_8); + private static final byte[] NEW = "NEW".getBytes(StandardCharsets.UTF_8); + + @TempDir java.nio.file.Path tempDir; + + private Target target; + private FileSystemCatalog catalog; + + @BeforeEach + void before() throws Exception { + target = new Target(); + catalog = + new FileSystemCatalog( + new TargetRetainingFileIO(target, false, TableFileIOView.NONE), + new Path(tempDir.resolve("warehouse").toUri().toString())); + catalog.createDatabase("db", true); + } + + @ParameterizedTest(name = "primary key {0}") + @ValueSource(booleans = {true, false}) + void testDescriptorFieldsOutsideTheTableAreReadFromTheOrigin(boolean primaryKey) + throws Exception { + Schema.Builder schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.BLOB()) + .option(CoreOptions.BLOB_DESCRIPTOR_FIELD.key(), "payload"); + if (primaryKey) { + schema.primaryKey("id").option(CoreOptions.BUCKET.key(), "1"); + } else { + schema.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_FIELD.key(), "payload"); + } + FileStoreTable table = createTable("descriptors", schema.build()); + Path source = replacedSource(); + + write(table, GenericRow.of(1, descriptorOf(LocalFileIO.create(), source))); + + List rows = read(table); + assertThat(rows.get(0).getBlob(1).toData()).isEqualTo(NEW); + assertThat(target.reads).doesNotContain(source.getName()); + // the data file the table wrote still uses the target + assertThat(target.reads).anyMatch(name -> name.startsWith("data-")); + } + + @Test + void testBlobSourcesOutsideTheTableAreCopiedFromTheOrigin() throws Exception { + FileStoreTable table = + createTable( + "managed", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.BLOB()) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_FIELD.key(), "payload") + .build()); + Path source = replacedSource(); + + // the writer reads the source through the table's FileIO and copies it into a .blob file + write(table, GenericRow.of(1, descriptorOf(table.fileIO(), source))); + + assertThat(target.reads).doesNotContain(source.getName()); + assertThat(read(table).get(0).getBlob(1).toData()).isEqualTo(NEW); + } + + @Test + void testDataFilesUnderExternalPathsAreReadFromTheOrigin() throws Exception { + String external = tempDir.resolve("external").toUri().toString(); + FileStoreTable table = + createTable( + "external", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), external) + .option( + CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(), + "round-robin") + .build()); + write(table, GenericRow.of(1)); + List externalFiles = new ArrayList<>(); + try (Stream files = Files.walk(tempDir.resolve("external"))) { + files.filter(Files::isRegularFile).forEach(externalFiles::add); + } + assertThat(externalFiles) + .anyMatch(file -> file.getFileName().toString().startsWith("data-")); + + target.reads.clear(); + assertThat(read(table)).hasSize(1); + for (java.nio.file.Path file : externalFiles) { + assertThat(target.reads).doesNotContain(file.getFileName().toString()); + } + } + + @Test + void testFilesOfOlderSchemaPrefixesUseTheTargetAfterAReload() throws Exception { + FileStoreTable table = + createTable( + "altered", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "older-") + .build()); + write(table, GenericRow.of(1)); + new FileSystemSchemaManager(table.fileIO(), table.location()) + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "newer-")); + write(table(table.name()), GenericRow.of(2)); + + target.reads.clear(); + // a fresh catalog knows only the newest schema until a file needs the older ones + before(); + assertThat(read(table(table.name()))).hasSize(2); + assertThat(target.reads).anyMatch(name -> name.startsWith("older-")); + assertThat(target.reads).anyMatch(name -> name.startsWith("newer-")); + assertThat(target.historyLoads).hasSize(1); + } + + @Test + void testBranchFilesUseTheTargetWithTheBranchPrefix() throws Exception { + FileStoreTable table = + createTable( + "branched", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "main-") + .build()); + table.createBranch("dev"); + new FileSystemSchemaManager(table.fileIO(), table.location(), "dev") + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "branch-")); + Identifier dev = new Identifier("db", table.name(), "dev"); + write((FileStoreTable) catalog.getTable(dev), GenericRow.of(1)); + + for (FileStoreTable branch : + new FileStoreTable[] { + (FileStoreTable) catalog.getTable(dev), + table(table.name()).switchToBranch("dev") + }) { + target.reads.clear(); + assertThat(read(branch)).hasSize(1); + assertThat(target.reads).anyMatch(name -> name.startsWith("branch-")); + } + } + + private FileStoreTable createTable(String name, Schema schema) throws Exception { + catalog.createTable(Identifier.create("db", name), schema, false); + return table(name); + } + + private FileStoreTable table(String name) throws Exception { + return (FileStoreTable) catalog.getTable(Identifier.create("db", name)); + } + + // Named like a data file and outside the table; the target keeps OLD, origin has NEW. + private Path replacedSource() throws IOException { + Path source = + new Path( + tempDir.resolve("import") + .resolve("data-" + UUID.randomUUID() + "-1.png") + .toUri() + .toString()); + LocalFileIO.create().writeFile(source, new String(NEW, StandardCharsets.UTF_8), false); + target.retained.put(source.toString(), OLD); + return source; + } + + private static Blob descriptorOf(FileIO fileIO, Path source) { + return Blob.fromDescriptor( + UriReader.fromFile(fileIO), new BlobDescriptor(source.toString(), 0, NEW.length)); + } + + private static void write(FileStoreTable table, InternalRow row) throws Exception { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(row); + commit.commit(write.prepareCommit()); + } + } + + private static List read(FileStoreTable table) throws Exception { + ReadBuilder builder = table.newReadBuilder(); + List rows = new ArrayList<>(); + builder.newRead().createReader(builder.newScan().plan()).forEachRemaining(rows::add); + return rows; + } + + /** What the io-cache target serves and keeps, shared by every view. */ + private static class Target implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Map retained = new ConcurrentHashMap<>(); + private final List reads = new CopyOnWriteArrayList<>(); + private final List historyLoads = new CopyOnWriteArrayList<>(); + } + + /** + * Local files whose reads of Paimon data files go to a target that keeps its first copy, routed + * with the table view the way JindoFileIO routes them. + */ + private static class TargetRetainingFileIO extends LocalFileIO { + + private static final long serialVersionUID = 1L; + + private static final String SUFFIX = + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + + "(-[0-9]+\\..+|\\.managed\\.blob)"; + private static final Pattern DATA_FILE_SUFFIX = Pattern.compile(SUFFIX); + private static final Pattern ANY_PREFIX_DATA_FILE = Pattern.compile(".*" + SUFFIX); + + private final Target target; + private final boolean originOnly; + private final TableFileIOView view; + private transient volatile List olderPrefixes; + + private TargetRetainingFileIO(Target target, boolean originOnly, TableFileIOView view) { + this.target = target; + this.originOnly = originOnly; + this.view = view; + } + + @Override + public FileIO originOnly() { + return originOnly ? this : new TargetRetainingFileIO(target, true, view); + } + + @Override + public FileIO withTableView(TableFileIOView tableView) { + TableFileIOView merged = view.merge(tableView); + return originOnly || merged.equals(view) + ? this + : new TargetRetainingFileIO(target, false, merged); + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + if (!routes(path)) { + return super.newInputStream(path); + } + target.reads.add(path.getName()); + byte[] bytes = target.retained.get(path.toString()); + if (bytes == null) { + bytes = readAll(path); + target.retained.put(path.toString(), bytes); + } + return new ByteArraySeekableStream(bytes); + } + + private byte[] readAll(Path path) throws IOException { + try (SeekableInputStream in = super.newInputStream(path)) { + byte[] bytes = new byte[(int) getFileStatus(path).getLen()]; + int read = 0; + while (read < bytes.length) { + int n = in.read(bytes, read, bytes.length - read); + if (n < 0) { + break; + } + read += n; + } + return bytes; + } + } + + private boolean routes(Path path) throws IOException { + if (originOnly + || (!view.roots().isEmpty() && !TableFileIOView.isUnder(view.roots(), path))) { + return false; + } + List prefixes = new ArrayList<>(view.filePrefixes()); + prefixes.add("data-"); + prefixes.add("changelog-"); + if (isDataFile(path.getName(), prefixes)) { + return true; + } + if (view.history() == null || !ANY_PREFIX_DATA_FILE.matcher(path.getName()).matches()) { + return false; + } + if (olderPrefixes == null) { + target.historyLoads.add(path.getName()); + olderPrefixes = view.history().filePrefixes(originOnly()); + } + return isDataFile(path.getName(), olderPrefixes); + } + + private static boolean isDataFile(String name, List prefixes) { + for (String prefix : prefixes) { + if (name.startsWith(prefix) + && DATA_FILE_SUFFIX.matcher(name.substring(prefix.length())).matches()) { + return true; + } + } + return false; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java new file mode 100644 index 000000000000..a370920de28a --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.iceberg.IcebergTable; +import org.apache.paimon.table.lance.LanceTable; +import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests that each table takes its io-cache view of the catalog FileIO. */ +class TableFileIOViewTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + void testTablesTakeTheirViewOfTheCatalogFileIO() throws Exception { + ViewFileIO fileIO = new ViewFileIO(false, TableFileIOView.NONE); + try (FileSystemCatalog catalog = + new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + // this catalog only loads Format Tables, so write the schema directly + Identifier format = Identifier.create("db", "format"); + new FileSystemSchemaManager(fileIO, catalog.getTableLocation(format)) + .createTable( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option( + CoreOptions.TYPE.key(), + TableType.FORMAT_TABLE.toString()) + .build()); + Identifier custom = Identifier.create("db", "custom"); + catalog.createTable( + custom, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "custom-") + .option(CoreOptions.CHANGELOG_FILE_PREFIX.key(), "cl-") + .build(), + false); + Identifier plain = Identifier.create("db", "plain"); + catalog.createTable( + plain, Schema.newBuilder().column("id", DataTypes.INT()).build(), false); + + // other writers may replace Format Table files in place + Table formatTable = catalog.getTable(format); + assertThat(viewOf(formatTable).originOnly).isTrue(); + assertThat(formatTable.copy(Collections.emptyMap()).fileIO()) + .isSameAs(formatTable.fileIO()); + + // a Paimon table owns its location and passes the file prefixes of its schema + FileStoreTable table = (FileStoreTable) catalog.getTable(custom); + TableFileIOView view = viewOf(table).view; + assertThat(viewOf(table).originOnly).isFalse(); + assertThat(view.roots()).containsExactly(root(table.location())); + assertThat(view.filePrefixes()).containsExactly("custom-", "cl-"); + assertThat(view.history()) + .isEqualTo( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "main")); + FileStoreTable copy = + table.copy( + Collections.singletonMap( + CoreOptions.DATA_FILE_PREFIX.key(), "dynamic-")); + assertThat(viewOf(copy).view.filePrefixes()) + .containsExactly("custom-", "cl-", "dynamic-"); + assertThat(table.copy(Collections.emptyMap()).fileIO()).isSameAs(table.fileIO()); + assertThat(table.copyWithLatestSchema().fileIO()).isSameAs(table.fileIO()); + + FileStoreTable plainTable = (FileStoreTable) catalog.getTable(plain); + assertThat(viewOf(plainTable).view.roots()) + .containsExactly(root(plainTable.location())); + assertThat(viewOf(plainTable).view.filePrefixes()).isEmpty(); + } + + ObjectTable objectTable = + ObjectTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "objects")) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(objectTable).originOnly).isTrue(); + IcebergTable icebergTable = + IcebergTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "iceberg")) + .rowType(RowType.of(DataTypes.INT())) + .partitionKeys(Collections.emptyList()) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(icebergTable).originOnly).isTrue(); + LanceTable lanceTable = + LanceTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "lance")) + .rowType(RowType.of(DataTypes.INT())) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(lanceTable).originOnly).isTrue(); + } + + @Test + void testBranchesAndExternalPathsTakeTheirOwnView() throws Exception { + ViewFileIO fileIO = new ViewFileIO(false, TableFileIOView.NONE); + String external = tempDir.resolve("external").toUri().toString(); + try (FileSystemCatalog catalog = + new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + Identifier identifier = Identifier.create("db", "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "main-") + .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), external) + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + // only the table's own location: its external paths read from origin + assertThat(viewOf(table).view.roots()).containsExactly(root(table.location())); + + table.createBranch("dev"); + FileStoreTable dev = table.switchToBranch("dev"); + assertThat(viewOf(dev).view.history()) + .isEqualTo( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "dev")); + new FileSystemSchemaManager(fileIO, table.location(), "dev") + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "branch-")); + FileStoreTable reloaded = + (FileStoreTable) catalog.getTable(new Identifier("db", "t", "dev")); + assertThat(viewOf(reloaded).view.filePrefixes()).containsExactly("branch-"); + assertThat(viewOf(table.switchToBranch("dev")).view.filePrefixes()).contains("branch-"); + + // the history of a branch lists its own schemas, from storage + assertThat( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "dev") + .filePrefixes(LocalFileIO.create())) + .containsExactly("main-", "branch-"); + } + } + + private static ViewFileIO viewOf(Table table) { + return (ViewFileIO) table.fileIO(); + } + + private static String root(Path path) { + return TableFileIOView.of(Collections.singletonList(path), Collections.emptyList(), null) + .roots() + .get(0); + } + + /** A FileIO that records the views tables take. */ + private static class ViewFileIO extends LocalFileIO { + + private static final long serialVersionUID = 1L; + + private final boolean originOnly; + private final TableFileIOView view; + + private ViewFileIO(boolean originOnly, TableFileIOView view) { + this.originOnly = originOnly; + this.view = view; + } + + @Override + public FileIO originOnly() { + return originOnly ? this : new ViewFileIO(true, view); + } + + @Override + public FileIO withTableView(TableFileIOView tableView) { + TableFileIOView merged = view.merge(tableView); + return originOnly || merged.equals(view) ? this : new ViewFileIO(false, merged); + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index d365e691efcd..5cabfcdc6693 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -85,7 +85,7 @@ public abstract class HadoopCompliantFileIO implements FileIO { boolean shouldCache(Path path) { if (cacheRouting != null) { - return cacheRouting.targetOf(path) != null; + return cacheTargetOf(path) != null; } if (cacheWhitelistPaths.isEmpty()) { return true; @@ -253,6 +253,19 @@ public boolean rename(Path src, Path dst) throws IOException { return getFileSystem(hadoopSrc, false).rename(hadoopSrc, hadoopDst); } + /** + * The cache target of a file, loading the prefixes of older schemas if only they may route it. + */ + @Nullable + IoCacheRouting.CacheTarget cacheTargetOf(Path path) { + IoCacheRouting.CacheTarget target = cacheRouting.targetOf(path); + if (target == null && cacheRouting.awaitsHistory(path)) { + cacheRouting.loadHistory(originOnly()); + target = cacheRouting.targetOf(path); + } + return target; + } + protected org.apache.hadoop.fs.Path path(Path path) { URI uri = path.toUri(); if ("oss".equals(uri.getScheme()) && uri.getUserInfo() != null) { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index 0576211f2075..88ec92bf0024 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -18,13 +18,20 @@ package org.apache.paimon.jindo; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; import org.apache.paimon.utils.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; +import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -56,6 +63,8 @@ final class IoCacheRouting implements Serializable { private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(IoCacheRouting.class); + private static final String META_CACHE_ENABLED_TAG = "meta"; private static final String READ_CACHE_ENABLED_TAG = "read"; private static final String WRITE_CACHE_ENABLED_TAG = "write"; @@ -72,6 +81,9 @@ final class IoCacheRouting implements Serializable { private static final Pattern DATA_FILE_SUFFIX = Pattern.compile( "(?:-u-.+-s-[0-9]+-w-)?" + UUID + "-[0-9]+\\..+|" + UUID + "\\.managed\\.blob"); + // The name of a data file or its file index, whatever its prefix. + private static final Pattern ANY_PREFIX_DATA_FILE = + Pattern.compile(".*(?:" + UUID + "-[0-9]+\\..+|" + UUID + "\\.managed\\.blob)"); // Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to // changelog manifests); Format Table files may be replaced in place. @@ -92,6 +104,9 @@ final class IoCacheRouting implements Serializable { private final List targets; private final Map targetByType; private final List dataFilePrefixes; + // Locations of the table a view serves; empty when this routing serves any table. + private final List roots; + @Nullable private final PrefixHistory history; private final boolean metaCacheEnabled; private final boolean readCacheEnabled; private final boolean writeCacheEnabled; @@ -102,6 +117,8 @@ private IoCacheRouting( List targets, Map targetByType, List dataFilePrefixes, + List roots, + @Nullable PrefixHistory history, boolean metaCacheEnabled, boolean readCacheEnabled, boolean writeCacheEnabled, @@ -110,6 +127,8 @@ private IoCacheRouting( this.targets = targets; this.targetByType = targetByType; this.dataFilePrefixes = dataFilePrefixes; + this.roots = roots; + this.history = history; this.metaCacheEnabled = metaCacheEnabled; this.readCacheEnabled = readCacheEnabled; this.writeCacheEnabled = writeCacheEnabled; @@ -170,12 +189,70 @@ static IoCacheRouting create(Options options) { new ArrayList<>(byName.values()), targetByType, dataFilePrefixes(options), + Collections.emptyList(), + null, metaCache, readCache, writeCache, existsCache); } + /** This routing with no request on a cache target. */ + IoCacheRouting originOnly() { + return targetByType.isEmpty() + ? this + : new IoCacheRouting( + ossEndpoint, + targets, + new EnumMap<>(FileType.class), + dataFilePrefixes, + roots, + null, + metaCacheEnabled, + readCacheEnabled, + writeCacheEnabled, + existsCacheEnabled); + } + + /** + * This routing for the table of {@code view}: only files under its roots may use a cache + * target, also when named with its prefixes or those of its older schemas. + */ + IoCacheRouting withTableView(TableFileIOView view) { + if (view.isOriginOnly()) { + return originOnly(); + } + if (targetByType.isEmpty() || view.isNone()) { + return this; + } + List mergedPrefixes = new ArrayList<>(dataFilePrefixes); + view.filePrefixes().forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); + List mergedRoots = new ArrayList<>(roots); + view.roots().forEach(root -> addIfAbsent(mergedRoots, root)); + PrefixHistory mergedHistory = + view.history() == null + ? history + : history != null && history.source.equals(view.history()) + ? history + : new PrefixHistory(view.history()); + if (mergedPrefixes.equals(dataFilePrefixes) + && mergedRoots.equals(roots) + && mergedHistory == history) { + return this; + } + return new IoCacheRouting( + ossEndpoint, + targets, + targetByType, + mergedPrefixes, + mergedRoots, + mergedHistory, + metaCacheEnabled, + readCacheEnabled, + writeCacheEnabled, + existsCacheEnabled); + } + private static Set parsePolicy(@Nullable String value) { return value == null ? Collections.emptySet() @@ -213,14 +290,53 @@ Collection targets() { /** The cache target selected for a file, or null when it does not use a cache target. */ @Nullable CacheTarget targetOf(Path path) { - // Only OSS paths can use cache targets. - if (!"oss".equals(path.toUri().getScheme())) { + // Only OSS paths can use cache targets, and a table view only its own files. + if (!"oss".equals(path.toUri().getScheme()) + || (!roots.isEmpty() && !TableFileIOView.isUnder(roots, path))) { return null; } - FileType type = cacheableType(path, dataFilePrefixes); + FileType type = cacheableType(path, filePrefixes()); return type == null ? null : targetByType.get(type); } + /** Whether only the prefixes of older schemas, not loaded yet, may route {@code path}. */ + boolean awaitsHistory(Path path) { + if (history == null + || history.loaded != null + || !(targetByType.containsKey(FileType.DATA) + || targetByType.containsKey(FileType.FILE_INDEX)) + || (!roots.isEmpty() && !TableFileIOView.isUnder(roots, path))) { + return false; + } + String name = path.getName(); + return !name.endsWith(MANIFEST_SIDECAR_SUFFIX) + && ANY_PREFIX_DATA_FILE.matcher(name).matches() + && !isDataFileName(name, dataFilePrefixes); + } + + /** Loads the prefixes of older schemas once, through a FileIO that only uses OSS. */ + void loadHistory(FileIO originOnly) { + if (history != null) { + history.load(originOnly); + } + } + + private List filePrefixes() { + List loaded = history == null ? null : history.loaded; + if (loaded == null || loaded.isEmpty()) { + return dataFilePrefixes; + } + List prefixes = new ArrayList<>(dataFilePrefixes); + loaded.forEach(prefix -> addIfAbsent(prefixes, prefix)); + return prefixes; + } + + private static void addIfAbsent(List list, String value) { + if (!StringUtils.isNullOrWhitespaceOnly(value) && !list.contains(value)) { + list.add(value); + } + } + /** The type of a file a cache may serve, or null when it must be read from OSS. */ @Nullable static FileType cacheableType(Path path, List dataFilePrefixes) { @@ -347,6 +463,32 @@ public String toString() { return "IoCacheRouting{" + targetByType + ", oss=" + ossEndpoint + "}"; } + /** The prefixes of older schemas, loaded at most once and shared by derived routings. */ + private static final class PrefixHistory implements Serializable { + + private static final long serialVersionUID = 1L; + + private final FilePrefixHistory source; + @Nullable private transient volatile List loaded; + + private PrefixHistory(FilePrefixHistory source) { + this.source = source; + } + + private synchronized void load(FileIO originOnly) { + if (loaded != null) { + return; + } + try { + loaded = new ArrayList<>(source.filePrefixes(originOnly)); + } catch (IOException | RuntimeException e) { + // files with a prefix missed here are read from OSS, which only loses the cache + LOG.warn("Failed to load the file prefixes of older schemas for io-cache.", e); + loaded = Collections.emptyList(); + } + } + } + /** A named cache target with its endpoint and addressing settings. */ static final class CacheTarget implements Serializable { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 0be4a335c7b3..2f9ea2b86e30 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.HadoopOptionsProvider; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; @@ -105,6 +106,8 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP transient volatile Map> cacheTargetFsMap; private boolean allowCache = true; private transient BlobPresigner blobPresigner; + // the FileIO a table view was made from; a view uses its file systems + @Nullable private JindoFileIO base; public JindoFileIO() {} @@ -192,6 +195,34 @@ public void configure(CatalogContext context) { } } + @Override + public FileIO originOnly() { + return cacheRouting == null ? this : withRouting(cacheRouting.originOnly()); + } + + @Override + public FileIO withTableView(TableFileIOView view) { + return cacheRouting == null ? this : withRouting(cacheRouting.withTableView(view)); + } + + private JindoFileIO withRouting(IoCacheRouting routing) { + if (routing == cacheRouting) { + return this; + } + JindoFileIO view = new JindoFileIO(); + view.base = base == null ? this : base; + view.cacheRouting = routing; + view.metaCacheEnabled = metaCacheEnabled; + view.readCacheEnabled = readCacheEnabled; + view.writeCacheEnabled = writeCacheEnabled; + view.existsCacheEnabled = existsCacheEnabled; + view.allowCache = allowCache; + view.hadoopOptions = hadoopOptions; + view.hadoopOptionsWithCache = hadoopOptionsWithCache; + view.cacheTargetOptions = cacheTargetOptions; + return view; + } + /** Copies the options with an endpoint: its host[:port], and https from its scheme if any. */ static Options withEndpoint(Options base, @Nullable String endpoint) { Options options = new Options(base.toMap()); @@ -237,7 +268,7 @@ Options routedHadoopOptions(Path path, String opType) { if (!shouldCache) { return hadoopOptions; } else if (cacheRouting != null) { - return cacheTargetOptions.get(cacheRouting.targetOf(path).name); + return cacheTargetOptions.get(cacheTargetOf(path).name); } else { return hadoopOptionsWithCache; } @@ -285,6 +316,9 @@ public String createBlobPresignedUrl( } private synchronized BlobPresigner blobPresigner() { + if (base != null) { + return base.blobPresigner(); + } if (blobPresigner == null) { PluginLoader loader = BlobPlugin.getLoader(); Thread thread = Thread.currentThread(); @@ -328,13 +362,21 @@ Pair createFileSystem( @Override protected Pair getFileSystemPair( org.apache.hadoop.fs.Path path, boolean enableCache) throws IOException { - if (!enableCache || cacheRouting == null) { - return super.getFileSystemPair(path, enableCache); + CacheTarget target = + enableCache && cacheRouting != null ? cacheTargetOf(new Path(path.toUri())) : null; + if (base != null) { + return target == null + ? base.getFileSystemPair(path, false) + : base.targetFileSystemPair(path, target); } - CacheTarget target = cacheRouting.targetOf(new Path(path.toUri())); if (target == null) { - return super.getFileSystemPair(path, false); + return super.getFileSystemPair(path, enableCache && cacheRouting == null); } + return targetFileSystemPair(path, target); + } + + private Pair targetFileSystemPair( + org.apache.hadoop.fs.Path path, CacheTarget target) throws IOException { if (cacheTargetFsMap == null) { synchronized (this) { if (cacheTargetFsMap == null) { @@ -398,6 +440,10 @@ private Pair createFileSystem( @Override public synchronized void close() { + if (base != null) { + // the file systems and the presigner belong to the base + return; + } if (blobPresigner != null) { Thread thread = Thread.currentThread(); ClassLoader previous = thread.getContextClassLoader(); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index 2dd9dfab9ce0..4ac070004037 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -19,7 +19,10 @@ package org.apache.paimon.jindo; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; @@ -30,7 +33,11 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import javax.annotation.Nullable; + +import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.apache.paimon.utils.FileType.BUCKET_INDEX; @@ -361,6 +368,115 @@ public void testCacheableTypeWithFilePrefixes() { assertType(warehouse + "branch/branch-feature", null, "data-file.prefix=branch-"); } + @Test + public void testTableViews() { + IoCacheRouting routing = IoCacheRouting.create(multi()); + Path custom = path("dt=1/bucket-0/custom-{uuid}-0.orc"); + assertThat(routing.targetOf(custom)).isNull(); + + // a table passes the file prefixes of its schema + IoCacheRouting prefixed = routing.withTableView(view(null, "custom-", " ")); + assertThat(prefixed.targetOf(custom).name).isEqualTo("cluster"); + assertThat(prefixed.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(prefixed.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(prefixed.withTableView(view(null, "custom-"))).isSameAs(prefixed); + assertThat(routing.withTableView(TableFileIOView.NONE)).isSameAs(routing); + + // Format and Object Tables send every request to the origin + IoCacheRouting originOnly = prefixed.originOnly(); + for (Path path : Arrays.asList(custom, DATA_FILE, MANIFEST, path(INDEX_PATH))) { + assertThat(originOnly.targetOf(path)).as(path.toString()).isNull(); + } + assertThat(originOnly.originOnly()).isSameAs(originOnly); + assertThat(originOnly.withTableView(view(null, "custom-"))).isSameAs(originOnly); + assertThat(prefixed.withTableView(view(null).originOnly()).targetOf(DATA_FILE)).isNull(); + assertThat(originOnly.ossEndpoint()).isEqualTo(OSS); + assertThat(originOnly.readCacheEnabled()).isTrue(); + } + + @Test + public void testTableViewsOnlyRouteFilesUnderTheirRoots() { + IoCacheRouting routing = IoCacheRouting.create(multi()); + // what a BLOB descriptor points at may be named like a data file and still be replaced + Path imported = path("oss://bkt/import/data-{uuid}-1.png"); + Path otherTable = path("oss://bkt/db1.db/t10/dt=1/bucket-0/data-{uuid}-0.parquet"); + Path otherBucket = path("oss://other/db1.db/t1/dt=1/bucket-0/data-{uuid}-0.parquet"); + assertThat(routing.targetOf(imported).name).isEqualTo("cluster"); + + IoCacheRouting table = routing.withTableView(view(null)); + for (Path path : Arrays.asList(imported, otherTable, otherBucket)) { + assertThat(table.targetOf(path)).as(path.toString()).isNull(); + } + assertThat(table.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(table.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(table.targetOf(path(TABLE_ROOT + "/"))).isNull(); + + // an external path on the table's bucket is one of its roots + IoCacheRouting external = + routing.withTableView( + TableFileIOView.of( + Arrays.asList(new Path(TABLE_ROOT), new Path("oss://bkt/import/")), + Collections.emptyList(), + null)); + assertThat(external.targetOf(imported).name).isEqualTo("cluster"); + assertThat(external.targetOf(otherTable)).isNull(); + } + + @Test + public void testPrefixesOfOlderSchemasAreLoadedOnceWhenOnlyTheyMayRoute() { + CountingHistory history = new CountingHistory("older-"); + IoCacheRouting routing = IoCacheRouting.create(multi()).withTableView(view(history)); + Path older = path("dt=1/bucket-0/older-{uuid}-0.parquet"); + assertThat(routing.targetOf(older)).isNull(); + + // known prefixes, metadata and files outside the table never load them + for (Path path : + Arrays.asList( + DATA_FILE, + MANIFEST, + path(SNAPSHOT_PATH), + path("manifest/manifest-{uuid}-0.avro.sidecar"), + path("oss://bkt/import/older-{uuid}-0.parquet"))) { + assertThat(routing.awaitsHistory(path)).as(path.toString()).isFalse(); + } + assertThat(routing.awaitsHistory(older)).isTrue(); + routing.loadHistory(null); + routing.loadHistory(null); + assertThat(history.loads).isEqualTo(1); + assertThat(routing.awaitsHistory(older)).isFalse(); + assertThat(routing.targetOf(older).name).isEqualTo("cluster"); + // a routing derived from it shares what was loaded + assertThat(routing.withTableView(view(null, "custom-")).targetOf(older).name) + .isEqualTo("cluster"); + + // a copy on another process loads them again + IoCacheRouting copy = clone(routing); + assertThat(copy.awaitsHistory(older)).isTrue(); + + // without a target for data files nothing is loaded + Options metaOnly = multi(); + metaOnly.set("io-cache.whitelist", "meta"); + assertThat( + IoCacheRouting.create(metaOnly) + .withTableView(view(history)) + .awaitsHistory(older)) + .isFalse(); + } + + @Test + public void testFailingToLoadOlderSchemasKeepsTheirFilesOnTheOrigin() { + FilePrefixHistory failing = + fileIO -> { + throw new IOException("schema directory not readable"); + }; + IoCacheRouting routing = IoCacheRouting.create(multi()).withTableView(view(failing)); + Path older = path("dt=1/bucket-0/older-{uuid}-0.parquet"); + routing.loadHistory(null); + assertThat(routing.awaitsHistory(older)).isFalse(); + assertThat(routing.targetOf(older)).isNull(); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + } + @Test public void testTargetNames() { Options options = singleTargetOptions(); @@ -567,6 +683,38 @@ private static void assertType(String path, FileType type, String... changes) { .isEqualTo(type); } + private static TableFileIOView view(@Nullable FilePrefixHistory history, String... prefixes) { + return TableFileIOView.of( + Collections.singletonList(new Path(TABLE_ROOT)), Arrays.asList(prefixes), history); + } + + private static IoCacheRouting clone(IoCacheRouting routing) { + try { + return InstantiationUtil.clone(routing, IoCacheRoutingTest.class.getClassLoader()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** Older schemas named their data files with these prefixes; counts the loads. */ + private static class CountingHistory implements FilePrefixHistory { + + private static final long serialVersionUID = 1L; + + private final List prefixes; + private int loads; + + private CountingHistory(String... prefixes) { + this.prefixes = Arrays.asList(prefixes); + } + + @Override + public List filePrefixes(FileIO fileIO) { + loads++; + return prefixes; + } + } + private static Path path(String path) { String resolved = path.replace("{uuid}", UUID); return new Path(resolved.contains("://") ? resolved : TABLE_ROOT + "/" + resolved); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 3426b9f85d98..6b7656e75b19 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -19,9 +19,13 @@ package org.apache.paimon.jindo; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; +import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; import com.aliyun.jindodata.api.spec.JdoException; @@ -41,6 +45,8 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -254,6 +260,49 @@ public void testWritesFollowTheRouteWithWritePolicy() throws IOException { verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + FileIO view = fileIO.originOnly(); + assertThat(view).isInstanceOf(JindoFileIO.class).isNotSameAs(fileIO); + assertThat(view.originOnly()).isSameAs(view); + assertThat(view.withTableView(view("custom-"))).isSameAs(view); + + view.newInputStream(DATA).close(); + view.getFileStatus(MANIFEST); + view.exists(DATA); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + verify(ossFs).exists(any()); + Options oss = fileIO.routedHadoopOptions(DATA, "write"); + assertThat(((JindoFileIO) view).routedHadoopOptions(DATA, "read")).isSameAs(oss); + assertThat(((JindoFileIO) view).routedHadoopOptions(MANIFEST, "meta")).isSameAs(oss); + + // the base keeps its routes + fileIO.newInputStream(DATA).close(); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(accelFs, never()).getFileStatus(any()); + } + + @Test + public void testFilePrefixViewRoutesDataFilesNamedWithThem() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path custom = new Path(TABLE + "/dt=1/bucket-0/custom-" + UUID + "-1.orc"); + assertThat(fileIO.withTableView(TableFileIOView.NONE)).isSameAs(fileIO); + + fileIO.newInputStream(custom).close(); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + + FileIO view = fileIO.withTableView(view("custom-")); + assertThat(view.withTableView(view("custom-"))).isSameAs(view); + view.newInputStream(custom).close(); + view.newInputStream(DATA).close(); + view.getFileStatus(MANIFEST); + verify(clusterFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(accelFs).getFileStatus(any()); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + } + @Test public void testPostponeFilesAndBlobPacksUseTheirTarget() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); @@ -265,6 +314,80 @@ public void testPostponeFilesAndBlobPacksUseTheirTarget() throws IOException { verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testTableViewSendsWhatItsRowsPointAtOutsideTheTableToOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + // a BLOB descriptor in a row of the table, named like a data file + Path imported = new Path("oss://bkt/import/data-" + UUID + "-1.png"); + FileIO view = fileIO.withTableView(view()); + + view.newInputStream(imported).close(); + view.getFileStatus(imported); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + view.newInputStream(DATA).close(); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + assertThat(((JindoFileIO) view).routedHadoopOptions(imported, "read")) + .isSameAs(fileIO.routedHadoopOptions(DATA, "write")); + } + + @Test + public void testPrefixesOfOlderSchemasAreReadFromOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path older = new Path(TABLE + "/dt=1/bucket-0/older-" + UUID + "-1.orc"); + List readers = new ArrayList<>(); + FilePrefixHistory history = + reader -> { + readers.add(reader); + reader.exists(new Path(TABLE + "/schema")); + return Collections.singletonList("older-"); + }; + FileIO view = + fileIO.withTableView( + TableFileIOView.of( + Collections.singletonList(new Path(TABLE)), + Collections.emptyList(), + history)); + + view.newInputStream(DATA).close(); + assertThat(readers).isEmpty(); + view.newInputStream(older).close(); + view.newInputStream(older).close(); + assertThat(readers).hasSize(1); + // the schemas are listed on OSS, the data file is read from its target + verify(ossFs).exists(any()); + verify(clusterFs, times(3)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testViewsUseTheFileSystemsOfTheirBase() throws Exception { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = configuredFileIO(options); + FileIO view = fileIO.withTableView(view("custom-")).originOnly(); + + view.newInputStream(DATA).close(); + assertThat(fileIO.fsMap).hasSize(1); + view.close(); + verify(ossFs, never()).close(); + fileIO.close(); + verify(ossFs).close(); + + // a serialized view keeps its routes + JindoFileIO plain = new JindoFileIO(); + plain.configure(CatalogContext.create(cacheTargetOptions())); + Path custom = new Path(TABLE + "/dt=1/bucket-0/custom-" + UUID + "-1.orc"); + JindoFileIO copy = + InstantiationUtil.clone( + (JindoFileIO) plain.withTableView(view("custom-")), + getClass().getClassLoader()); + assertThat(copy.routedHadoopOptions(custom, "read").get("fs.oss.endpoint")) + .isEqualTo(CLUSTER_HOST); + assertThat(plain.routedHadoopOptions(custom, "read").get("fs.oss.endpoint")) + .isEqualTo(OSS_HOST); + } + @Test public void testSingleCacheTarget() { Options options = baseOptions(); @@ -313,6 +436,8 @@ public void testWithoutIoCacheRouting() { JindoFileIO fileIO = new JindoFileIO(); fileIO.configure(CatalogContext.create(options)); assertThat(fileIO.cacheRouting).isNull(); + assertThat(fileIO.originOnly()).isSameAs(fileIO); + assertThat(fileIO.withTableView(view("custom-"))).isSameAs(fileIO); for (String op : new String[] {"read", "meta", "write"}) { Options hadoopOptions = fileIO.routedHadoopOptions(DATA, op); assertThat(hadoopOptions.get("fs.oss.endpoint")) @@ -428,6 +553,11 @@ private static Options cacheTargetOptions() { return options; } + private static TableFileIOView view(String... prefixes) { + return TableFileIOView.of( + Collections.singletonList(new Path(TABLE)), Arrays.asList(prefixes), null); + } + private JindoFileIO configuredFileIO(Options options) { JindoFileIO fileIO = new TestingJindoFileIO(accelFs, clusterFs, ossFs); fileIO.configure(CatalogContext.create(options)); From 6e90d3ae5c57011ca2b308050085fbf3bf5ffd0c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 03:19:52 +0800 Subject: [PATCH 12/15] [core] Keep io-cache views off FileIO, so forwarding FileIOs stay as given The previous commit added originOnly() and withTableView() to FileIO as default methods returning this. A FileIO decorator that forwards every method, such as the java.lang.reflect.Proxy wrappers in Spark's FormatTablePartitionManagementTest, forwards these defaults as well: the delegate returned itself, so a Format Table got the bare delegate instead of the decorator, and the decorator's injected failures were lost. Any wrapper that forwards all FileIO methods would drop itself the same way, and a default method on the public FileIO interface cannot tell a wrapper from the FileIO it wraps. FileIO is unchanged now. The two methods move to IoCacheViewableFileIO, implemented by the FileIOs that have io-cache routing or wrap one: JindoFileIO, RESTTokenFileIO, ResolvingFileIO and CachingFileIO, which forward to a delegate only when it implements the interface. Every caller goes through IoCacheViews.originOnly() and IoCacheViews.withTableView(), which return any other FileIO as it is, so Format, Object, Iceberg and Lance tables and every FileStoreTable keep a decorator they are given. TableFileIOViewTest gives a forwarding Proxy to a Format Table, an Object Table and a FileSystemCatalog table and checks each keeps that instance; it fails on the previous commit. The same test compared the prefixes of a branch's schemas in listing order, which is the file system's: LocalFileIO lists the schema directory in File.list order, which differs between machines. Routing only checks whether a prefix is known, so the test now ignores the order. --- .../java/org/apache/paimon/fs/FileIO.java | 10 ---- .../paimon/fs/IoCacheViewableFileIO.java | 32 ++++++++++++ .../org/apache/paimon/fs/IoCacheViews.java | 39 ++++++++++++++ .../org/apache/paimon/fs/ResolvingFileIO.java | 2 +- .../org/apache/paimon/fs/TableFileIOView.java | 6 +-- .../apache/paimon/fs/cache/CachingFileIO.java | 8 +-- .../apache/paimon/rest/RESTTokenFileIO.java | 3 +- .../apache/paimon/utils/UriReaderFactory.java | 3 +- .../apache/paimon/fs/ResolvingFileIOTest.java | 4 +- .../paimon/fs/cache/CachingFileIOTest.java | 10 +++- .../paimon/rest/RESTTokenFileIOTest.java | 8 +-- .../paimon/utils/UriReaderFactoryTest.java | 3 +- .../org/apache/paimon/table/FormatTable.java | 3 +- .../apache/paimon/table/TableFileIOViews.java | 4 +- .../paimon/table/iceberg/IcebergTable.java | 3 +- .../apache/paimon/table/lance/LanceTable.java | 3 +- .../paimon/table/object/ObjectTable.java | 3 +- .../paimon/table/IoCacheTableRoutingTest.java | 4 +- .../paimon/table/TableFileIOViewTest.java | 52 ++++++++++++++++++- .../paimon/jindo/HadoopCompliantFileIO.java | 3 +- .../org/apache/paimon/jindo/JindoFileIO.java | 4 +- .../paimon/jindo/JindoIoCacheRoutingTest.java | 9 ++-- 22 files changed, 176 insertions(+), 40 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index af47af570c91..16efe5f6a627 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -80,16 +80,6 @@ public interface FileIO extends Serializable, Closeable { /** Set filesystem options at runtime. Usually used for job-level settings. */ default void setRuntimeContext(Map options) {} - /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ - default FileIO originOnly() { - return this; - } - - /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ - default FileIO withTableView(TableFileIOView view) { - return this; - } - /** * Opens an SeekableInputStream at the indicated Path. * diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java new file mode 100644 index 000000000000..887eda4c8b15 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +/** + * A {@link FileIO} whose io-cache routing can be narrowed: to the origin, or to the files of one + * table. Call it through {@link IoCacheViews}, which leaves every other FileIO as it is. + */ +public interface IoCacheViewableFileIO extends FileIO { + + /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ + FileIO originOnly(); + + /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ + FileIO withTableView(TableFileIOView view); +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java new file mode 100644 index 000000000000..bc5ab18259bf --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +/** Applies io-cache views to a FileIO that supports them, and returns any other FileIO as is. */ +public final class IoCacheViews { + + private IoCacheViews() {} + + /** See {@link IoCacheViewableFileIO#originOnly()}. */ + public static FileIO originOnly(FileIO fileIO) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).originOnly() + : fileIO; + } + + /** See {@link IoCacheViewableFileIO#withTableView(TableFileIOView)}. */ + public static FileIO withTableView(FileIO fileIO, TableFileIOView view) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).withTableView(view) + : fileIO; + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java index 4f8cd210dcf0..c81859720324 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java @@ -38,7 +38,7 @@ * An implementation of {@link FileIO} that supports multiple file system schemas. It dynamically * selects the appropriate {@link FileIO} based on the URI scheme of the given path. */ -public class ResolvingFileIO implements FileIO { +public class ResolvingFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java b/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java index 7f14eeae6fc5..e6903ca6a94e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java @@ -81,7 +81,7 @@ public static TableFileIOView of( history); } - /** See {@link FileIO#originOnly()}. */ + /** See {@link IoCacheViewableFileIO#originOnly()}. */ public TableFileIOView originOnly() { return ORIGIN_ONLY; } @@ -142,9 +142,9 @@ public static boolean isUnder(List roots, Path path) { public FileIO apply(FileIO fileIO) { if (originOnly) { - return fileIO.originOnly(); + return IoCacheViews.originOnly(fileIO); } - return isNone() ? fileIO : fileIO.withTableView(this); + return isNone() ? fileIO : IoCacheViews.withTableView(fileIO, this); } private static String normalizeRoot(Path path) { diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java index 0e3e49afcad4..6c1ebc6722a7 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java @@ -22,6 +22,8 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.IoCacheViewableFileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; @@ -54,7 +56,7 @@ *

After deserialization, copies of the same catalog-created wrapper lazily reuse a JVM-local * cache manager. */ -public class CachingFileIO implements FileIO { +public class CachingFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; @@ -219,12 +221,12 @@ public void setRuntimeContext(Map options) { @Override public FileIO originOnly() { - return withDelegate(delegate.originOnly()); + return withDelegate(IoCacheViews.originOnly(delegate)); } @Override public FileIO withTableView(TableFileIOView view) { - return withDelegate(delegate.withTableView(view)); + return withDelegate(IoCacheViews.withTableView(delegate, view)); } // a view shares this wrapper's cache diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index 1690b8027132..5d47b56d2b7b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; @@ -61,7 +62,7 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; /** A {@link FileIO} to support getting token from REST Server. */ -public class RESTTokenFileIO implements FileIO { +public class RESTTokenFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 2L; diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java index 866d9f7bf467..8d365c887832 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import javax.annotation.Nullable; @@ -97,7 +98,7 @@ protected UriReader newReader(URI uri) { try { FileIO createdFileIO = FileIO.get(new Path(uri), Objects.requireNonNull(context)); // no table owns what a descriptor points at here, and its writer may replace it - return UriReader.fromFile(createdFileIO.originOnly()); + return UriReader.fromFile(IoCacheViews.originOnly(createdFileIO)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java index 0f63bf58368e..1d0a0b256f24 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java @@ -60,7 +60,7 @@ public void setUp() { @Test public void testTableViewsApplyToEachResolvedFileIO() throws IOException { - FileIO oss = mock(FileIO.class); + IoCacheViewableFileIO oss = mock(IoCacheViewableFileIO.class); FileIO originOnly = mock(FileIO.class); when(oss.originOnly()).thenReturn(originOnly); FileIOLoader loader = mock(FileIOLoader.class); @@ -72,7 +72,7 @@ public void testTableViewsApplyToEachResolvedFileIO() throws IOException { FileIO view = fileIO.originOnly(); assertInstanceOf(ResolvingFileIO.class, view); - assertSame(view, view.originOnly()); + assertSame(view, IoCacheViews.originOnly(view)); assertSame(originOnly, ((ResolvingFileIO) view).fileIO(data)); assertSame(oss, fileIO.fileIO(data)); // loading a FileIO checks access with exists diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index 84aabb5d7c76..5a103462b997 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileRange; import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; @@ -220,7 +221,7 @@ private CachingFileIO newCachingFileIO( @Test void testTableViewsWrapTheViewOfTheDelegate() throws IOException { - FileIO delegate = mock(FileIO.class); + IoCacheViewableFileIO delegate = mock(IoCacheViewableFileIO.class); FileIO originOnly = mock(FileIO.class); when(delegate.originOnly()).thenReturn(originOnly); when(delegate.withTableView(any())).thenReturn(delegate); @@ -243,6 +244,13 @@ void testTableViewsWrapTheViewOfTheDelegate() throws IOException { Collections.singletonList("custom-"), null); assertThat(cachingIO.withTableView(tableView)).isSameAs(cachingIO); + CachingFileIO plain = + new CachingFileIO( + mock(FileIO.class), + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META)); + assertThat(plain.originOnly()).isSameAs(plain); + assertThat(plain.withTableView(tableView)).isSameAs(plain); } @Test diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java index ac2ca9ff7e73..ea3b41e549b7 100644 --- a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -24,6 +24,8 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.IoCacheViewableFileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.TableFileIOView; @@ -260,7 +262,7 @@ void testFileIOCreationFailureSurfacesAsCheckedIOException() throws IOException @Test void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { Path tableRoot = new Path("oss://bucket/table"); - FileIO delegate = mock(FileIO.class); + IoCacheViewableFileIO delegate = mock(IoCacheViewableFileIO.class); FileIO originOnly = mock(FileIO.class); FileIO prefixed = mock(FileIO.class); TableFileIOView tableView = @@ -289,14 +291,14 @@ void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { FileIO prefixView = fileIO.withTableView(tableView); assertThat(prefixView).isInstanceOf(RESTTokenFileIO.class).isNotSameAs(fileIO); - assertThat(prefixView.withTableView(tableView)).isSameAs(prefixView); + assertThat(IoCacheViews.withTableView(prefixView, tableView)).isSameAs(prefixView); assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); // the view of the token's FileIO is built once verify(delegate, times(1)).withTableView(any()); FileIO originView = fileIO.originOnly(); - assertThat(originView.originOnly()).isSameAs(originView); + assertThat(IoCacheViews.originOnly(originView)).isSameAs(originView); Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); originView.exists(data); verify(originOnly).exists(data); diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java index 94992505bc91..7440eca25161 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.IsolatedDirectoryFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; @@ -76,7 +77,7 @@ public void tearDownHttpServer() { @Test public void testFilesOfDescriptorsAreReadFromTheOriginOfTheCreatedFileIO() throws Exception { // a descriptor may name a file another writer replaces, and no table owns it here - FileIO created = mock(FileIO.class); + IoCacheViewableFileIO created = mock(IoCacheViewableFileIO.class); FileIO originOnly = mock(FileIO.class); when(created.originOnly()).thenReturn(originOnly); FileIOLoader loader = mock(FileIOLoader.class); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 8c8b03bad950..032925f9dca9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -26,6 +26,7 @@ import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; @@ -188,7 +189,7 @@ public Builder partitionManager(@Nullable FormatTablePartitionManager partitionM public FormatTable build() { // other writers may replace Format Table files in place return new FormatTableImpl( - fileIO == null ? null : fileIO.originOnly(), + fileIO == null ? null : IoCacheViews.originOnly(fileIO), identifier, rowType, partitionKeys, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java index 7411ac5a071a..db31bd8aeafb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FilePrefixHistory; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.schema.FileSystemSchemaManager; @@ -47,7 +48,8 @@ static FileIO forTable(FileIO fileIO, Path path, Map options) { prefixes.add(options.get(DATA_FILE_PREFIX.key())); prefixes.add(options.get(CHANGELOG_FILE_PREFIX.key())); // only files under the table's location use io-cache, not those under external paths - return fileIO.withTableView( + return IoCacheViews.withTableView( + fileIO, TableFileIOView.of( Collections.singletonList(path), prefixes, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java index 1df00a20dc89..89e95a36d81e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -96,7 +97,7 @@ public IcebergTable build() { // other engines write Iceberg tables, so their files never use io-cache return new IcebergTableImpl( identifier, - fileIO == null ? null : fileIO.originOnly(), + fileIO == null ? null : IoCacheViews.originOnly(fileIO), rowType, partitionKeys, location, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java index 0c5a2f9cd63a..487bf9209cf3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -85,7 +86,7 @@ public LanceTable build() { // other engines write Lance datasets, so their files never use io-cache return new LanceTableImpl( identifier, - fileIO == null ? null : fileIO.originOnly(), + fileIO == null ? null : IoCacheViews.originOnly(fileIO), rowType, location, options, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java index 6b22232d31a4..576588c35288 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -94,7 +95,7 @@ public ObjectTable build() { // other writers manage the objects, so they may change under the same name return new ObjectTableImpl( identifier, - fileIO == null ? null : fileIO.originOnly(), + fileIO == null ? null : IoCacheViews.originOnly(fileIO), location, options, comment); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java index fa1b06b6ef59..2a80fc37937e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.TableFileIOView; @@ -271,7 +272,8 @@ private static class Target implements Serializable { * Local files whose reads of Paimon data files go to a target that keeps its first copy, routed * with the table view the way JindoFileIO routes them. */ - private static class TargetRetainingFileIO extends LocalFileIO { + private static class TargetRetainingFileIO extends LocalFileIO + implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java index a370920de28a..c308e1e61fce 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.local.LocalFileIO; @@ -38,6 +39,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; @@ -172,7 +175,52 @@ void testBranchesAndExternalPathsTakeTheirOwnView() throws Exception { assertThat( new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "dev") .filePrefixes(LocalFileIO.create())) - .containsExactly("main-", "branch-"); + .containsExactlyInAnyOrder("main-", "branch-"); + } + } + + @Test + void testFileIOsWithoutViewsStayTheSameInstance() throws Exception { + // a decorator that forwards every FileIO method, default ones included + LocalFileIO local = LocalFileIO.create(); + FileIO forwarding = + (FileIO) + Proxy.newProxyInstance( + FileIO.class.getClassLoader(), + new Class[] {FileIO.class}, + (proxy, method, args) -> { + try { + return method.invoke(local, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + FormatTable formatTable = + FormatTable.builder() + .fileIO(forwarding) + .identifier(Identifier.create("db", "format")) + .rowType(RowType.of(DataTypes.INT())) + .partitionKeys(Collections.emptyList()) + .location(tempDir.resolve("format").toUri().toString()) + .format(FormatTable.Format.PARQUET) + .options(Collections.emptyMap()) + .build(); + assertThat(formatTable.fileIO()).isSameAs(forwarding); + ObjectTable objectTable = + ObjectTable.builder() + .fileIO(forwarding) + .identifier(Identifier.create("db", "objects")) + .location(tempDir.resolve("objects").toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(objectTable.fileIO()).isSameAs(forwarding); + try (FileSystemCatalog catalog = + new FileSystemCatalog(forwarding, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + Identifier identifier = Identifier.create("db", "t"); + catalog.createTable( + identifier, Schema.newBuilder().column("id", DataTypes.INT()).build(), false); + assertThat(catalog.getTable(identifier).fileIO()).isSameAs(forwarding); } } @@ -187,7 +235,7 @@ private static String root(Path path) { } /** A FileIO that records the views tables take. */ - private static class ViewFileIO extends LocalFileIO { + private static class ViewFileIO extends LocalFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 5cabfcdc6693..fe33673c05cf 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; @@ -260,7 +261,7 @@ public boolean rename(Path src, Path dst) throws IOException { IoCacheRouting.CacheTarget cacheTargetOf(Path path) { IoCacheRouting.CacheTarget target = cacheRouting.targetOf(path); if (target == null && cacheRouting.awaitsHistory(path)) { - cacheRouting.loadHistory(originOnly()); + cacheRouting.loadHistory(IoCacheViews.originOnly(this)); target = cacheRouting.targetOf(path); } return target; diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 2f9ea2b86e30..09e4800975a1 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.HadoopOptionsProvider; +import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.TwoPhaseOutputStream; @@ -58,7 +59,8 @@ import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE; /** Jindo {@link FileIO}. */ -public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsProvider { +public class JindoFileIO extends HadoopCompliantFileIO + implements HadoopOptionsProvider, IoCacheViewableFileIO { private static final long serialVersionUID = 2L; diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 6b7656e75b19..d2e08625323d 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FilePrefixHistory; +import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; @@ -265,8 +266,8 @@ public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); FileIO view = fileIO.originOnly(); assertThat(view).isInstanceOf(JindoFileIO.class).isNotSameAs(fileIO); - assertThat(view.originOnly()).isSameAs(view); - assertThat(view.withTableView(view("custom-"))).isSameAs(view); + assertThat(IoCacheViews.originOnly(view)).isSameAs(view); + assertThat(IoCacheViews.withTableView(view, view("custom-"))).isSameAs(view); view.newInputStream(DATA).close(); view.getFileStatus(MANIFEST); @@ -294,7 +295,7 @@ public void testFilePrefixViewRoutesDataFilesNamedWithThem() throws IOException verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); FileIO view = fileIO.withTableView(view("custom-")); - assertThat(view.withTableView(view("custom-"))).isSameAs(view); + assertThat(IoCacheViews.withTableView(view, view("custom-"))).isSameAs(view); view.newInputStream(custom).close(); view.newInputStream(DATA).close(); view.getFileStatus(MANIFEST); @@ -365,7 +366,7 @@ public void testViewsUseTheFileSystemsOfTheirBase() throws Exception { Options options = cacheTargetOptions(); options.set(FILE_IO_ALLOW_CACHE, false); JindoFileIO fileIO = configuredFileIO(options); - FileIO view = fileIO.withTableView(view("custom-")).originOnly(); + FileIO view = IoCacheViews.originOnly(fileIO.withTableView(view("custom-"))); view.newInputStream(DATA).close(); assertThat(fileIO.fsMap).hasSize(1); From 84363ffb61fd3fe816c56337dd251b10fbf81c58 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 20:45:37 +0800 Subject: [PATCH 13/15] [core] Scope the cached FileIO of a REST token to the catalog options it was built from RESTTokenFileIO keeps the FileIO of each vended token in a static cache, so every RESTTokenFileIO in the JVM that holds the same token shares it. The FileIO is built from the catalog options merged with the token, and io-cache routing may come from the catalog options alone: io-cache.origin.endpoint, the targets, the policy, the whitelist and the routes. Two catalogs with one token but different catalog-only routing therefore shared the first one's JindoFileIO, and the second read and wrote through the first one's origin and cache. The cache key is now the token together with an immutable copy of the catalog options, taken once per RESTTokenFileIO, and the FileIO is built from that same copy. The catalog's dlf.oss-endpoint and explicit io-cache.enabled are already merged into the token at refresh. Copies of one catalog's RESTTokenFileIO, including those deserialized on workers, still share one FileIO. RESTTokenFileIOTest checks that the same token under other catalog options gets its own FileIO. JindoIoCacheRoutingTest builds two RESTTokenFileIOs with one token and different catalog-only origin and target endpoints through the real REST cache and JindoFileIO configuration, and checks that each sends reads and writes to its own endpoints and gives Lance its own origin. Both fail without the change. --- .../apache/paimon/rest/RESTTokenFileIO.java | 63 +++++++++++++++++-- .../paimon/rest/RESTTokenFileIOTest.java | 36 +++++++++++ .../paimon/jindo/JindoIoCacheRoutingTest.java | 52 +++++++++++++++ 3 files changed, 145 insertions(+), 6 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index 5d47b56d2b7b..105cef475879 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -51,6 +51,8 @@ import java.io.IOException; import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -72,7 +74,7 @@ public class RESTTokenFileIO implements IoCacheViewableFileIO { .defaultValue(false) .withDescription("Whether to support data token provided by the REST server."); - private static final Cache FILE_IO_CACHE = + private static final Cache FILE_IO_CACHE = Caffeine.newBuilder() .maximumSize(1000) .expireAfterAccess(10, TimeUnit.HOURS) @@ -121,6 +123,8 @@ static long fileIOCacheMaximumSize() { private final TableFileIOView view; private transient volatile Pair lastView; + private transient volatile FileIOKey lastKey; + public RESTTokenFileIO( CatalogContext catalogContext, RESTApi apiInstance, Identifier identifier, Path path) { this(catalogContext, apiInstance, identifier, path, TableFileIOView.NONE, null); @@ -295,19 +299,19 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc + "REST credential lifetime after refresh."); } - FileIO fileIO = FILE_IO_CACHE.getIfPresent(currentToken); + FileIOKey key = keyOf(currentToken); + FileIO fileIO = FILE_IO_CACHE.getIfPresent(key); if (fileIO != null) { return new FileIOWithToken(viewOf(fileIO), currentToken); } synchronized (FILE_IO_CACHE) { - fileIO = FILE_IO_CACHE.getIfPresent(currentToken); + fileIO = FILE_IO_CACHE.getIfPresent(key); if (fileIO != null) { return new FileIOWithToken(viewOf(fileIO), currentToken); } - Options options = catalogContext.options(); - options = new Options(RESTUtil.merge(options.toMap(), currentToken.token())); + Options options = new Options(RESTUtil.merge(key.catalogOptions, currentToken.token())); options.set(FILE_IO_ALLOW_CACHE, false); CatalogContext context = CatalogContext.create( @@ -316,11 +320,25 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc catalogContext.preferIO(), catalogContext.fallbackIO()); fileIO = FileIO.get(path, context); - FILE_IO_CACHE.put(currentToken, fileIO); + FILE_IO_CACHE.put(key, fileIO); return new FileIOWithToken(viewOf(fileIO), currentToken); } } + private FileIOKey keyOf(RESTToken currentToken) { + FileIOKey key = lastKey; + if (key == null || key.token != currentToken) { + Map catalogOptions = + key == null + ? Collections.unmodifiableMap( + new HashMap<>(catalogContext.options().toMap())) + : key.catalogOptions; + key = new FileIOKey(currentToken, catalogOptions); + lastKey = key; + } + return key; + } + private boolean hasRemainingLifetime(RESTToken signingToken, long minimumValidityMillis) { return signingToken == null || signingToken.expireAtMillis() - currentTimeMillis() >= minimumValidityMillis; @@ -350,6 +368,39 @@ private boolean shouldRefresh(long minimumValidityMillis) { < Math.max(TOKEN_EXPIRATION_SAFE_TIME_MILLIS, minimumValidityMillis); } + /** A cached FileIO serves one token with the catalog options it was built from. */ + private static final class FileIOKey { + + private final RESTToken token; + private final Map catalogOptions; + private final int hash; + + private FileIOKey(RESTToken token, Map catalogOptions) { + this.token = token; + this.catalogOptions = catalogOptions; + this.hash = 31 * token.hashCode() + catalogOptions.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FileIOKey)) { + return false; + } + FileIOKey that = (FileIOKey) o; + return hash == that.hash + && token.equals(that.token) + && catalogOptions.equals(that.catalogOptions); + } + + @Override + public int hashCode() { + return hash; + } + } + private static class FileIOWithToken { private final FileIO fileIO; diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java index ea3b41e549b7..d4f527e20182 100644 --- a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -306,6 +306,42 @@ void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { assertThat(fileIO.fileIO()).isSameAs(delegate); } + @Test + void testFileIOsOfOneTokenAreCachedPerCatalogOptions() throws IOException { + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenAnswer(invocation -> mock(FileIO.class)); + when(loader.getScheme()).thenReturn("oss"); + Identifier identifier = Identifier.create("db", "table"); + GetTableTokenResponse response = + new GetTableTokenResponse( + Collections.singletonMap("token", UUID.randomUUID().toString()), + Long.MAX_VALUE); + + FileIO first = tokenFileIO(loader, identifier, response, "http://cache-a:80").fileIO(); + FileIO same = tokenFileIO(loader, identifier, response, "http://cache-a:80").fileIO(); + FileIO other = tokenFileIO(loader, identifier, response, "http://cache-b:80").fileIO(); + + // catalog-only options change the backend, so they scope the cached FileIO too + assertThat(same).isSameAs(first); + assertThat(other).isNotSameAs(first); + } + + private static RESTTokenFileIO tokenFileIO( + FileIOLoader loader, + Identifier identifier, + GetTableTokenResponse response, + String cacheEndpoint) { + RESTApi api = mock(RESTApi.class); + when(api.loadTableToken(identifier)).thenReturn(response); + Options options = new Options(); + options.set("io-cache.endpoint", cacheEndpoint); + return new RESTTokenFileIO( + CatalogContext.create(options, loader, null), + api, + identifier, + new Path("oss://bucket/table")); + } + @Test void testTryToWriteAtomicReachesInnerOverride() throws IOException { Path tableRoot = new Path("oss://bucket/table"); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index d2e08625323d..68e90623ef90 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.jindo; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.IoCacheViews; @@ -26,6 +27,9 @@ import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTApi; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.rest.responses.GetTableTokenResponse; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; @@ -48,7 +52,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE; @@ -261,6 +267,52 @@ public void testWritesFollowTheRouteWithWritePolicy() throws IOException { verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); } + @Test + public void testTokenFileIOsOfOneTokenKeepTheirCatalogEndpoints() throws IOException { + Identifier identifier = Identifier.create("db1", "t1"); + Map token = new HashMap<>(); + token.put("fs.oss.accessKeyId", "ak-" + java.util.UUID.randomUUID()); + token.put("fs.oss.accessKeySecret", "sk"); + // the cache older clients use + token.put("fs.oss.endpoint", ACCEL_HOST); + GetTableTokenResponse response = new GetTableTokenResponse(token, Long.MAX_VALUE); + String otherOss = "oss-cn-shanghai-internal.aliyuncs.com"; + String otherCluster = "10.0.0.2:8080"; + + JindoFileIO first = jindoOf(tokenFileIO(identifier, response, OSS_HOST, CLUSTER_HOST)); + JindoFileIO second = jindoOf(tokenFileIO(identifier, response, otherOss, otherCluster)); + + assertThat(endpoint(first.routedHadoopOptions(DATA, "read"))).isEqualTo(CLUSTER_HOST); + assertThat(endpoint(first.routedHadoopOptions(DATA, "write"))).isEqualTo(OSS_HOST); + assertThat(endpoint(second.routedHadoopOptions(DATA, "read"))).isEqualTo(otherCluster); + assertThat(endpoint(second.routedHadoopOptions(DATA, "write"))).isEqualTo(otherOss); + assertThat(endpoint(second.hadoopOptions(DATA, "read"))).isEqualTo(otherOss); + assertThat(jindoOf(tokenFileIO(identifier, response, OSS_HOST, CLUSTER_HOST))) + .isSameAs(first); + } + + private static RESTTokenFileIO tokenFileIO( + Identifier identifier, GetTableTokenResponse response, String origin, String target) { + RESTApi api = mock(RESTApi.class); + when(api.loadTableToken(identifier)).thenReturn(response); + Options options = new Options(); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", origin); + options.set("io-cache.endpoint", "http://" + target); + options.set("io-cache.target.default.path-style-access", "true"); + options.set("io-cache.policy", "meta,read"); + return new RESTTokenFileIO( + CatalogContext.create(options), api, identifier, new Path(TABLE)); + } + + private static JindoFileIO jindoOf(RESTTokenFileIO fileIO) throws IOException { + return (JindoFileIO) fileIO.fileIO(); + } + + private static String endpoint(Options options) { + return options.get("fs.oss.endpoint"); + } + @Test public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); From 88561b9761013875a992f9781e1264364d67d2a3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 00:20:53 +0800 Subject: [PATCH 14/15] [core] Cover ROW sidecars and the older prefixes a branch inherits in io-cache tests A review of the Rust client found two requests that missed their target: the size check of a ROW sidecar went to origin while its read used the target, and a REST branch whose schema only the catalog keeps knew no older prefix of the files it inherited from main. Neither happens here. DataEvolutionSplitRead sizes the sidecar with getFileStatus on the table's FileIO, which follows the meta rule, and a branch reads every older schema from its own schema directory, which SchemaFilePrefixHistory already lists; without those schemas the read itself fails before any data request. These tests keep both: a ROW sidecar is classified as data and sized and read through the target, and a branch created from a tag reads the files it inherited under main's older prefix through the target. --- .../paimon/table/IoCacheTableRoutingTest.java | 26 +++++++++++++++++++ .../paimon/jindo/IoCacheRoutingTest.java | 5 ++++ 2 files changed, 31 insertions(+) diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java index 2a80fc37937e..76a8c4dfbefe 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java @@ -215,6 +215,32 @@ void testBranchFilesUseTheTargetWithTheBranchPrefix() throws Exception { } } + @Test + void testBranchFilesOfOlderMainPrefixesUseTheTarget() throws Exception { + FileStoreTable table = + createTable( + "inherited", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "older-") + .build()); + write(table, GenericRow.of(1)); + new FileSystemSchemaManager(table.fileIO(), table.location()) + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "newer-")); + write(table(table.name()), GenericRow.of(2)); + table(table.name()).createTag("tag"); + table(table.name()).createBranch("dev", "tag"); + + // the branch keeps copies of the schemas its files were written with + before(); + Identifier dev = new Identifier("db", table.name(), "dev"); + target.reads.clear(); + assertThat(read((FileStoreTable) catalog.getTable(dev))).hasSize(2); + assertThat(target.reads).anyMatch(name -> name.startsWith("older-")); + assertThat(target.reads).anyMatch(name -> name.startsWith("newer-")); + } + private FileStoreTable createTable(String name, Schema schema) throws Exception { catalog.createTable(Identifier.create("db", name), schema, false); return table(name); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index 4ac070004037..e27183a51eb5 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -77,6 +77,9 @@ public void testOneCacheTarget() { Options options = single(); assertEndpoint(options, "read", DATA_PATH, CACHE); assertEndpoint(options, "meta", DATA_PATH, CACHE); + // a ROW sidecar is sized and read through the target of its data file + assertEndpoint(single("io-cache.whitelist=data"), "meta", DATA_PATH + ".row", CACHE); + assertEndpoint(single("io-cache.whitelist=data"), "read", DATA_PATH + ".row", CACHE); // exists asks whether a file is still there, so it needs its own policy token assertEndpoint(options, "exists", DATA_PATH, OSS); assertEndpoint(single(EXISTS), "exists", DATA_PATH, CACHE); @@ -304,6 +307,7 @@ public void testCacheableType() { assertType("dt=1/8b1f7c2e/data-{uuid}-0.parquet", DATA); assertType("dt=1/bucket-0/data-{uuid}-0.parquet.index", FILE_INDEX); assertType("dt=1/bucket-0/data-{uuid}-0.parquet.blobref", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.row", DATA); assertType("dt=1/bucket-0/data-{uuid}-0.managed.blob", DATA); assertType("dt=1/bucket-0/data-{uuid}.managed.blob", DATA); // the postpone writer puts its commit user and write id before {uuid}-{count} @@ -315,6 +319,7 @@ public void testCacheableType() { assertType(postpone, DATA); assertType(postpone + ".index", FILE_INDEX); assertType(postpone + ".blobref", DATA); + assertType(postpone + ".row", DATA); } assertType("index/index-{uuid}-0", BUCKET_INDEX); assertType("index/btree-global-index-{uuid}.index", GLOBAL_INDEX); From 39e3428fa265717fd2c7de87528b90ee8adcbc88 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 00:28:46 +0800 Subject: [PATCH 15/15] [common] Keep the io-cache view types in one internal interface The view support added four public types to org.apache.paimon.fs: IoCacheViewableFileIO, IoCacheViews, TableFileIOView and FilePrefixHistory. They exist only for io-cache routing, so they now live in one interface, org.apache.paimon.fs.iocache.IoCacheViewableFileIO, marked as internal: the static helpers of IoCacheViews become its static methods, and TableFileIOView and FilePrefixHistory become its nested TableView and PrefixHistory. org.apache.paimon.fs gains no type and FileIO is unchanged. Behavior is the same; every user and test moves to the new names. --- .../apache/paimon/fs/FilePrefixHistory.java | 30 --- .../paimon/fs/IoCacheViewableFileIO.java | 32 --- .../org/apache/paimon/fs/IoCacheViews.java | 39 --- .../org/apache/paimon/fs/ResolvingFileIO.java | 8 +- .../org/apache/paimon/fs/TableFileIOView.java | 190 --------------- .../apache/paimon/fs/cache/CachingFileIO.java | 11 +- .../fs/iocache/IoCacheViewableFileIO.java | 228 ++++++++++++++++++ .../apache/paimon/rest/RESTTokenFileIO.java | 14 +- .../apache/paimon/utils/UriReaderFactory.java | 4 +- .../apache/paimon/fs/ResolvingFileIOTest.java | 3 +- .../paimon/fs/cache/CachingFileIOTest.java | 8 +- .../paimon/rest/RESTTokenFileIOTest.java | 13 +- .../paimon/utils/UriReaderFactoryTest.java | 2 +- .../org/apache/paimon/table/FormatTable.java | 4 +- .../apache/paimon/table/TableFileIOViews.java | 14 +- .../paimon/table/iceberg/IcebergTable.java | 4 +- .../apache/paimon/table/lance/LanceTable.java | 4 +- .../paimon/table/object/ObjectTable.java | 4 +- .../paimon/table/IoCacheTableRoutingTest.java | 17 +- .../paimon/table/TableFileIOViewTest.java | 20 +- .../paimon/jindo/HadoopCompliantFileIO.java | 4 +- .../apache/paimon/jindo/IoCacheRouting.java | 14 +- .../org/apache/paimon/jindo/JindoFileIO.java | 6 +- .../paimon/jindo/IoCacheRoutingTest.java | 16 +- .../paimon/jindo/JindoIoCacheRoutingTest.java | 24 +- 25 files changed, 325 insertions(+), 388 deletions(-) delete mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java delete mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java delete mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java delete mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java b/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java deleted file mode 100644 index 1cc50d02f3e7..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FilePrefixHistory.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.fs; - -import java.io.IOException; -import java.io.Serializable; -import java.util.List; - -/** Loads the data file prefixes of the schemas a table has had, for io-cache routing. */ -public interface FilePrefixHistory extends Serializable { - - /** Reads them through {@code fileIO}, which sends no request to an io-cache target. */ - List filePrefixes(FileIO fileIO) throws IOException; -} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java deleted file mode 100644 index 887eda4c8b15..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViewableFileIO.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.fs; - -/** - * A {@link FileIO} whose io-cache routing can be narrowed: to the origin, or to the files of one - * table. Call it through {@link IoCacheViews}, which leaves every other FileIO as it is. - */ -public interface IoCacheViewableFileIO extends FileIO { - - /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ - FileIO originOnly(); - - /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ - FileIO withTableView(TableFileIOView view); -} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java b/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java deleted file mode 100644 index bc5ab18259bf..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/fs/IoCacheViews.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.fs; - -/** Applies io-cache views to a FileIO that supports them, and returns any other FileIO as is. */ -public final class IoCacheViews { - - private IoCacheViews() {} - - /** See {@link IoCacheViewableFileIO#originOnly()}. */ - public static FileIO originOnly(FileIO fileIO) { - return fileIO instanceof IoCacheViewableFileIO - ? ((IoCacheViewableFileIO) fileIO).originOnly() - : fileIO; - } - - /** See {@link IoCacheViewableFileIO#withTableView(TableFileIOView)}. */ - public static FileIO withTableView(FileIO fileIO, TableFileIOView view) { - return fileIO instanceof IoCacheViewableFileIO - ? ((IoCacheViewableFileIO) fileIO).withTableView(view) - : fileIO; - } -} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java index c81859720324..8728e61dfc33 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java @@ -20,6 +20,8 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -48,7 +50,7 @@ public class ResolvingFileIO implements IoCacheViewableFileIO { // a table view resolves through its base and applies the view to each FileIO @Nullable private ResolvingFileIO base; - private TableFileIOView view = TableFileIOView.NONE; + private TableView view = TableView.NONE; @Override public FileIO originOnly() { @@ -56,11 +58,11 @@ public FileIO originOnly() { } @Override - public FileIO withTableView(TableFileIOView tableView) { + public FileIO withTableView(TableView tableView) { return withView(view.merge(tableView)); } - private FileIO withView(TableFileIOView newView) { + private FileIO withView(TableView newView) { if (newView == view) { return this; } diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java b/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java deleted file mode 100644 index e6903ca6a94e..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/fs/TableFileIOView.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.fs; - -import javax.annotation.Nullable; - -import java.io.Serializable; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Objects; - -/** - * What io-cache routing knows about the table a FileIO view serves: the locations it owns, the - * prefixes of its data files and where to find the prefixes of its older schemas. - */ -public final class TableFileIOView implements Serializable { - - private static final long serialVersionUID = 2L; - - public static final TableFileIOView NONE = - new TableFileIOView(false, Collections.emptyList(), Collections.emptyList(), null); - - private static final TableFileIOView ORIGIN_ONLY = - new TableFileIOView(true, Collections.emptyList(), Collections.emptyList(), null); - - private final boolean originOnly; - private final List roots; - private final List filePrefixes; - @Nullable private final FilePrefixHistory history; - - private TableFileIOView( - boolean originOnly, - List roots, - List filePrefixes, - @Nullable FilePrefixHistory history) { - this.originOnly = originOnly; - this.roots = roots; - this.filePrefixes = filePrefixes; - this.history = history; - } - - /** The view of a table whose files are under {@code roots}. */ - public static TableFileIOView of( - Collection roots, - Collection filePrefixes, - @Nullable FilePrefixHistory history) { - List normalizedRoots = new ArrayList<>(); - for (Path root : roots) { - addIfAbsent(normalizedRoots, normalizeRoot(root)); - } - List prefixes = new ArrayList<>(); - for (String prefix : filePrefixes) { - if (prefix != null && !prefix.trim().isEmpty()) { - addIfAbsent(prefixes, prefix); - } - } - return new TableFileIOView( - false, - Collections.unmodifiableList(normalizedRoots), - Collections.unmodifiableList(prefixes), - history); - } - - /** See {@link IoCacheViewableFileIO#originOnly()}. */ - public TableFileIOView originOnly() { - return ORIGIN_ONLY; - } - - /** This view also serving {@code other}'s table, with its history; origin only stays so. */ - public TableFileIOView merge(TableFileIOView other) { - if (originOnly || other.isNone()) { - return this; - } - if (other.originOnly || isNone()) { - return other; - } - List mergedRoots = new ArrayList<>(roots); - other.roots.forEach(root -> addIfAbsent(mergedRoots, root)); - List mergedPrefixes = new ArrayList<>(filePrefixes); - other.filePrefixes.forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); - TableFileIOView merged = - new TableFileIOView( - false, - Collections.unmodifiableList(mergedRoots), - Collections.unmodifiableList(mergedPrefixes), - other.history == null ? history : other.history); - return merged.equals(this) ? this : merged; - } - - public boolean isOriginOnly() { - return originOnly; - } - - public boolean isNone() { - return this == NONE || (!originOnly && roots.isEmpty() && filePrefixes.isEmpty()); - } - - /** Normalized {@code scheme://authority/path} of each location the table owns. */ - public List roots() { - return roots; - } - - public List filePrefixes() { - return filePrefixes; - } - - @Nullable - public FilePrefixHistory history() { - return history; - } - - /** Whether {@code path} is one of the roots or under one. */ - public static boolean isUnder(List roots, Path path) { - String location = normalizeRoot(path); - for (String root : roots) { - if (location.equals(root) || location.startsWith(root + "/")) { - return true; - } - } - return false; - } - - public FileIO apply(FileIO fileIO) { - if (originOnly) { - return IoCacheViews.originOnly(fileIO); - } - return isNone() ? fileIO : IoCacheViews.withTableView(fileIO, this); - } - - private static String normalizeRoot(Path path) { - URI uri = path.toUri(); - String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); - String authority = uri.getAuthority() == null ? "" : uri.getAuthority(); - String location = uri.getPath() == null ? "" : uri.getPath(); - while (location.endsWith("/")) { - location = location.substring(0, location.length() - 1); - } - return scheme + "://" + authority + location; - } - - private static void addIfAbsent(List list, String value) { - if (!list.contains(value)) { - list.add(value); - } - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof TableFileIOView)) { - return false; - } - TableFileIOView that = (TableFileIOView) o; - return originOnly == that.originOnly - && roots.equals(that.roots) - && filePrefixes.equals(that.filePrefixes) - && Objects.equals(history, that.history); - } - - @Override - public int hashCode() { - return Objects.hash(originOnly, roots, filePrefixes, history); - } - - @Override - public String toString() { - return originOnly - ? "TableFileIOView{originOnly}" - : "TableFileIOView{roots=" + roots + ", prefixes=" + filePrefixes + "}"; - } -} diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java index 6c1ebc6722a7..12099267012d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java @@ -22,13 +22,12 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; -import org.apache.paimon.fs.IoCacheViewableFileIO; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -221,12 +220,12 @@ public void setRuntimeContext(Map options) { @Override public FileIO originOnly() { - return withDelegate(IoCacheViews.originOnly(delegate)); + return withDelegate(IoCacheViewableFileIO.originOnly(delegate)); } @Override - public FileIO withTableView(TableFileIOView view) { - return withDelegate(IoCacheViews.withTableView(delegate, view)); + public FileIO withTableView(TableView view) { + return withDelegate(IoCacheViewableFileIO.withTableView(delegate, view)); } // a view shares this wrapper's cache diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java new file mode 100644 index 000000000000..21be723de480 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs.iocache; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.Serializable; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * A {@link FileIO} whose io-cache routing can be narrowed to the origin or to one table's files. + * Internal to io-cache routing, not a public API. + */ +public interface IoCacheViewableFileIO extends FileIO { + + /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ + FileIO originOnly(); + + /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ + FileIO withTableView(TableView view); + + /** {@link #originOnly()} of a FileIO that supports views; any other FileIO as is. */ + static FileIO originOnly(FileIO fileIO) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).originOnly() + : fileIO; + } + + /** {@link #withTableView(TableView)} of a FileIO that supports views; any other as is. */ + static FileIO withTableView(FileIO fileIO, TableView view) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).withTableView(view) + : fileIO; + } + + /** Loads the data file prefixes of the schemas a table has had. */ + interface PrefixHistory extends Serializable { + + /** Reads them through {@code fileIO}, which sends no request to an io-cache target. */ + List filePrefixes(FileIO fileIO) throws IOException; + } + + /** + * What io-cache routing knows about the table a FileIO view serves: the locations it owns, the + * prefixes of its data files and where to find the prefixes of its older schemas. + */ + final class TableView implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final TableView NONE = + new TableView(false, Collections.emptyList(), Collections.emptyList(), null); + + private static final TableView ORIGIN_ONLY = + new TableView(true, Collections.emptyList(), Collections.emptyList(), null); + + private final boolean originOnly; + private final List roots; + private final List filePrefixes; + @Nullable private final PrefixHistory history; + + private TableView( + boolean originOnly, + List roots, + List filePrefixes, + @Nullable PrefixHistory history) { + this.originOnly = originOnly; + this.roots = roots; + this.filePrefixes = filePrefixes; + this.history = history; + } + + /** The view of a table whose files are under {@code roots}. */ + public static TableView of( + Collection roots, + Collection filePrefixes, + @Nullable PrefixHistory history) { + List normalizedRoots = new ArrayList<>(); + for (Path root : roots) { + addIfAbsent(normalizedRoots, normalizeRoot(root)); + } + List prefixes = new ArrayList<>(); + for (String prefix : filePrefixes) { + if (prefix != null && !prefix.trim().isEmpty()) { + addIfAbsent(prefixes, prefix); + } + } + return new TableView( + false, + Collections.unmodifiableList(normalizedRoots), + Collections.unmodifiableList(prefixes), + history); + } + + /** See {@link IoCacheViewableFileIO#originOnly()}. */ + public TableView originOnly() { + return ORIGIN_ONLY; + } + + /** This view also serving {@code other}'s table, with its history; origin only stays so. */ + public TableView merge(TableView other) { + if (originOnly || other.isNone()) { + return this; + } + if (other.originOnly || isNone()) { + return other; + } + List mergedRoots = new ArrayList<>(roots); + other.roots.forEach(root -> addIfAbsent(mergedRoots, root)); + List mergedPrefixes = new ArrayList<>(filePrefixes); + other.filePrefixes.forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); + TableView merged = + new TableView( + false, + Collections.unmodifiableList(mergedRoots), + Collections.unmodifiableList(mergedPrefixes), + other.history == null ? history : other.history); + return merged.equals(this) ? this : merged; + } + + public boolean isOriginOnly() { + return originOnly; + } + + public boolean isNone() { + return this == NONE || (!originOnly && roots.isEmpty() && filePrefixes.isEmpty()); + } + + /** Normalized {@code scheme://authority/path} of each location the table owns. */ + public List roots() { + return roots; + } + + public List filePrefixes() { + return filePrefixes; + } + + @Nullable + public PrefixHistory history() { + return history; + } + + /** Whether {@code path} is one of the roots or under one. */ + public static boolean isUnder(List roots, Path path) { + String location = normalizeRoot(path); + for (String root : roots) { + if (location.equals(root) || location.startsWith(root + "/")) { + return true; + } + } + return false; + } + + public FileIO apply(FileIO fileIO) { + if (originOnly) { + return IoCacheViewableFileIO.originOnly(fileIO); + } + return isNone() ? fileIO : IoCacheViewableFileIO.withTableView(fileIO, this); + } + + private static String normalizeRoot(Path path) { + URI uri = path.toUri(); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + String authority = uri.getAuthority() == null ? "" : uri.getAuthority(); + String location = uri.getPath() == null ? "" : uri.getPath(); + while (location.endsWith("/")) { + location = location.substring(0, location.length() - 1); + } + return scheme + "://" + authority + location; + } + + private static void addIfAbsent(List list, String value) { + if (!list.contains(value)) { + list.add(value); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof TableView)) { + return false; + } + TableView that = (TableView) o; + return originOnly == that.originOnly + && roots.equals(that.roots) + && filePrefixes.equals(that.filePrefixes) + && Objects.equals(history, that.history); + } + + @Override + public int hashCode() { + return Objects.hash(originOnly, roots, filePrefixes, history); + } + + @Override + public String toString() { + return originOnly + ? "TableView{originOnly}" + : "TableView{roots=" + roots + ", prefixes=" + filePrefixes + "}"; + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index 105cef475879..f06c55420c3d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -23,13 +23,13 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; import org.apache.paimon.options.Options; @@ -120,14 +120,14 @@ static long fileIOCacheMaximumSize() { private volatile RESTToken token; // applied to the FileIO of each token - private final TableFileIOView view; + private final TableView view; private transient volatile Pair lastView; private transient volatile FileIOKey lastKey; public RESTTokenFileIO( CatalogContext catalogContext, RESTApi apiInstance, Identifier identifier, Path path) { - this(catalogContext, apiInstance, identifier, path, TableFileIOView.NONE, null); + this(catalogContext, apiInstance, identifier, path, TableView.NONE, null); } private RESTTokenFileIO( @@ -135,7 +135,7 @@ private RESTTokenFileIO( RESTApi apiInstance, Identifier identifier, Path path, - TableFileIOView view, + TableView view, @Nullable RESTToken token) { this.catalogContext = catalogContext; this.apiInstance = apiInstance; @@ -151,11 +151,11 @@ public FileIO originOnly() { } @Override - public FileIO withTableView(TableFileIOView tableView) { + public FileIO withTableView(TableView tableView) { return withView(view.merge(tableView)); } - private FileIO withView(TableFileIOView newView) { + private FileIO withView(TableView newView) { return newView == view ? this : new RESTTokenFileIO( diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java index 8d365c887832..bfc1dcf9e0b1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java @@ -20,8 +20,8 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import javax.annotation.Nullable; @@ -98,7 +98,7 @@ protected UriReader newReader(URI uri) { try { FileIO createdFileIO = FileIO.get(new Path(uri), Objects.requireNonNull(context)); // no table owns what a descriptor points at here, and its writer may replace it - return UriReader.fromFile(IoCacheViews.originOnly(createdFileIO)); + return UriReader.fromFile(IoCacheViewableFileIO.originOnly(createdFileIO)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java index 1d0a0b256f24..e59e46e39b1b 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.hadoop.HadoopFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; @@ -72,7 +73,7 @@ public void testTableViewsApplyToEachResolvedFileIO() throws IOException { FileIO view = fileIO.originOnly(); assertInstanceOf(ResolvingFileIO.class, view); - assertSame(view, IoCacheViews.originOnly(view)); + assertSame(view, IoCacheViewableFileIO.originOnly(view)); assertSame(originOnly, ((ResolvingFileIO) view).fileIO(data)); assertSame(oss, fileIO.fileIO(data)); // loading a FileIO checks access with exists diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index 5a103462b997..74bbe46dc0cf 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -23,13 +23,13 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileRange; import org.apache.paimon.fs.FileStatus; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.VectoredReadUtils; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; @@ -238,8 +238,8 @@ void testTableViewsWrapTheViewOfTheDelegate() throws IOException { verify(originOnly).exists(data); verify(delegate, never()).exists(any()); // a delegate without anything to narrow keeps the wrapper - TableFileIOView tableView = - TableFileIOView.of( + TableView tableView = + TableView.of( Collections.singletonList(new Path("oss://bucket/t")), Collections.singletonList("custom-"), null); diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java index d4f527e20182..a35297426716 100644 --- a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -24,11 +24,10 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.FileStatus; -import org.apache.paimon.fs.IoCacheViewableFileIO; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.RemoteIterator; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.Options; import org.apache.paimon.rest.responses.GetTableTokenResponse; @@ -265,8 +264,8 @@ void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { IoCacheViewableFileIO delegate = mock(IoCacheViewableFileIO.class); FileIO originOnly = mock(FileIO.class); FileIO prefixed = mock(FileIO.class); - TableFileIOView tableView = - TableFileIOView.of( + TableView tableView = + TableView.of( Collections.singletonList(tableRoot), Collections.singletonList("custom-"), null); @@ -291,14 +290,14 @@ void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { FileIO prefixView = fileIO.withTableView(tableView); assertThat(prefixView).isInstanceOf(RESTTokenFileIO.class).isNotSameAs(fileIO); - assertThat(IoCacheViews.withTableView(prefixView, tableView)).isSameAs(prefixView); + assertThat(IoCacheViewableFileIO.withTableView(prefixView, tableView)).isSameAs(prefixView); assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); // the view of the token's FileIO is built once verify(delegate, times(1)).withTableView(any()); FileIO originView = fileIO.originOnly(); - assertThat(IoCacheViews.originOnly(originView)).isSameAs(originView); + assertThat(IoCacheViewableFileIO.originOnly(originView)).isSameAs(originView); Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); originView.exists(data); verify(originOnly).exists(data); diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java index 7440eca25161..d42cd95833bb 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java @@ -21,10 +21,10 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.IsolatedDirectoryFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.utils.UriReader.FileUriReader; import org.apache.paimon.utils.UriReader.HttpUriReader; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 032925f9dca9..b3521d5c6dcf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -26,7 +26,7 @@ import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViews; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; @@ -189,7 +189,7 @@ public Builder partitionManager(@Nullable FormatTablePartitionManager partitionM public FormatTable build() { // other writers may replace Format Table files in place return new FormatTableImpl( - fileIO == null ? null : IoCacheViews.originOnly(fileIO), + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), identifier, rowType, partitionKeys, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java index db31bd8aeafb..f61386f83489 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java @@ -20,10 +20,10 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.FilePrefixHistory; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.utils.StringUtils; @@ -37,7 +37,7 @@ import static org.apache.paimon.CoreOptions.CHANGELOG_FILE_PREFIX; import static org.apache.paimon.CoreOptions.DATA_FILE_PREFIX; -/** Builds the {@link TableFileIOView} every {@link FileStoreTable} applies to its FileIO. */ +/** Builds the {@link TableView} every {@link FileStoreTable} applies to its FileIO. */ final class TableFileIOViews { private TableFileIOViews() {} @@ -48,16 +48,16 @@ static FileIO forTable(FileIO fileIO, Path path, Map options) { prefixes.add(options.get(DATA_FILE_PREFIX.key())); prefixes.add(options.get(CHANGELOG_FILE_PREFIX.key())); // only files under the table's location use io-cache, not those under external paths - return IoCacheViews.withTableView( + return IoCacheViewableFileIO.withTableView( fileIO, - TableFileIOView.of( + TableView.of( Collections.singletonList(path), prefixes, new SchemaFilePrefixHistory(path, CoreOptions.branch(options)))); } /** The data and changelog file prefixes of every schema of a branch, read from storage. */ - static final class SchemaFilePrefixHistory implements FilePrefixHistory { + static final class SchemaFilePrefixHistory implements PrefixHistory { private static final long serialVersionUID = 1L; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java index 89e95a36d81e..0cbbb0fe3fd3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java @@ -20,7 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViews; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -97,7 +97,7 @@ public IcebergTable build() { // other engines write Iceberg tables, so their files never use io-cache return new IcebergTableImpl( identifier, - fileIO == null ? null : IoCacheViews.originOnly(fileIO), + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), rowType, partitionKeys, location, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java index 487bf9209cf3..d46ff7583406 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java @@ -20,7 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViews; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -86,7 +86,7 @@ public LanceTable build() { // other engines write Lance datasets, so their files never use io-cache return new LanceTableImpl( identifier, - fileIO == null ? null : IoCacheViews.originOnly(fileIO), + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), rowType, location, options, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java index 576588c35288..ee123bedaa23 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java @@ -20,7 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViews; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -95,7 +95,7 @@ public ObjectTable build() { // other writers manage the objects, so they may change under the same name return new ObjectTableImpl( identifier, - fileIO == null ? null : IoCacheViews.originOnly(fileIO), + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), location, options, comment); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java index 76a8c4dfbefe..ca17df122691 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java @@ -27,10 +27,10 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; @@ -82,7 +82,7 @@ void before() throws Exception { target = new Target(); catalog = new FileSystemCatalog( - new TargetRetainingFileIO(target, false, TableFileIOView.NONE), + new TargetRetainingFileIO(target, false, TableView.NONE), new Path(tempDir.resolve("warehouse").toUri().toString())); catalog.createDatabase("db", true); } @@ -311,10 +311,10 @@ private static class TargetRetainingFileIO extends LocalFileIO private final Target target; private final boolean originOnly; - private final TableFileIOView view; + private final TableView view; private transient volatile List olderPrefixes; - private TargetRetainingFileIO(Target target, boolean originOnly, TableFileIOView view) { + private TargetRetainingFileIO(Target target, boolean originOnly, TableView view) { this.target = target; this.originOnly = originOnly; this.view = view; @@ -326,8 +326,8 @@ public FileIO originOnly() { } @Override - public FileIO withTableView(TableFileIOView tableView) { - TableFileIOView merged = view.merge(tableView); + public FileIO withTableView(TableView tableView) { + TableView merged = view.merge(tableView); return originOnly || merged.equals(view) ? this : new TargetRetainingFileIO(target, false, merged); @@ -363,8 +363,7 @@ private byte[] readAll(Path path) throws IOException { } private boolean routes(Path path) throws IOException { - if (originOnly - || (!view.roots().isEmpty() && !TableFileIOView.isUnder(view.roots(), path))) { + if (originOnly || (!view.roots().isEmpty() && !TableView.isUnder(view.roots(), path))) { return false; } List prefixes = new ArrayList<>(view.filePrefixes()); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java index c308e1e61fce..6bd12bdc1347 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java @@ -23,9 +23,9 @@ import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; @@ -52,7 +52,7 @@ class TableFileIOViewTest { @Test void testTablesTakeTheirViewOfTheCatalogFileIO() throws Exception { - ViewFileIO fileIO = new ViewFileIO(false, TableFileIOView.NONE); + ViewFileIO fileIO = new ViewFileIO(false, TableView.NONE); try (FileSystemCatalog catalog = new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { catalog.createDatabase("db", false); @@ -87,7 +87,7 @@ void testTablesTakeTheirViewOfTheCatalogFileIO() throws Exception { // a Paimon table owns its location and passes the file prefixes of its schema FileStoreTable table = (FileStoreTable) catalog.getTable(custom); - TableFileIOView view = viewOf(table).view; + TableView view = viewOf(table).view; assertThat(viewOf(table).originOnly).isFalse(); assertThat(view.roots()).containsExactly(root(table.location())); assertThat(view.filePrefixes()).containsExactly("custom-", "cl-"); @@ -140,7 +140,7 @@ void testTablesTakeTheirViewOfTheCatalogFileIO() throws Exception { @Test void testBranchesAndExternalPathsTakeTheirOwnView() throws Exception { - ViewFileIO fileIO = new ViewFileIO(false, TableFileIOView.NONE); + ViewFileIO fileIO = new ViewFileIO(false, TableView.NONE); String external = tempDir.resolve("external").toUri().toString(); try (FileSystemCatalog catalog = new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { @@ -229,7 +229,7 @@ private static ViewFileIO viewOf(Table table) { } private static String root(Path path) { - return TableFileIOView.of(Collections.singletonList(path), Collections.emptyList(), null) + return TableView.of(Collections.singletonList(path), Collections.emptyList(), null) .roots() .get(0); } @@ -240,9 +240,9 @@ private static class ViewFileIO extends LocalFileIO implements IoCacheViewableFi private static final long serialVersionUID = 1L; private final boolean originOnly; - private final TableFileIOView view; + private final TableView view; - private ViewFileIO(boolean originOnly, TableFileIOView view) { + private ViewFileIO(boolean originOnly, TableView view) { this.originOnly = originOnly; this.view = view; } @@ -253,8 +253,8 @@ public FileIO originOnly() { } @Override - public FileIO withTableView(TableFileIOView tableView) { - TableFileIOView merged = view.merge(tableView); + public FileIO withTableView(TableView tableView) { + TableView merged = view.merge(tableView); return originOnly || merged.equals(view) ? this : new ViewFileIO(false, merged); } } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index fe33673c05cf..71e51347401d 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -21,12 +21,12 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.utils.Pair; @@ -261,7 +261,7 @@ public boolean rename(Path src, Path dst) throws IOException { IoCacheRouting.CacheTarget cacheTargetOf(Path path) { IoCacheRouting.CacheTarget target = cacheRouting.targetOf(path); if (target == null && cacheRouting.awaitsHistory(path)) { - cacheRouting.loadHistory(IoCacheViews.originOnly(this)); + cacheRouting.loadHistory(IoCacheViewableFileIO.originOnly(this)); target = cacheRouting.targetOf(path); } return target; diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java index 88ec92bf0024..b3d9f3c13ef0 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -19,9 +19,9 @@ package org.apache.paimon.jindo; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; import org.apache.paimon.utils.StringUtils; @@ -218,7 +218,7 @@ IoCacheRouting originOnly() { * This routing for the table of {@code view}: only files under its roots may use a cache * target, also when named with its prefixes or those of its older schemas. */ - IoCacheRouting withTableView(TableFileIOView view) { + IoCacheRouting withTableView(TableView view) { if (view.isOriginOnly()) { return originOnly(); } @@ -292,7 +292,7 @@ Collection targets() { CacheTarget targetOf(Path path) { // Only OSS paths can use cache targets, and a table view only its own files. if (!"oss".equals(path.toUri().getScheme()) - || (!roots.isEmpty() && !TableFileIOView.isUnder(roots, path))) { + || (!roots.isEmpty() && !TableView.isUnder(roots, path))) { return null; } FileType type = cacheableType(path, filePrefixes()); @@ -305,7 +305,7 @@ boolean awaitsHistory(Path path) { || history.loaded != null || !(targetByType.containsKey(FileType.DATA) || targetByType.containsKey(FileType.FILE_INDEX)) - || (!roots.isEmpty() && !TableFileIOView.isUnder(roots, path))) { + || (!roots.isEmpty() && !TableView.isUnder(roots, path))) { return false; } String name = path.getName(); @@ -468,10 +468,10 @@ private static final class PrefixHistory implements Serializable { private static final long serialVersionUID = 1L; - private final FilePrefixHistory source; + private final IoCacheViewableFileIO.PrefixHistory source; @Nullable private transient volatile List loaded; - private PrefixHistory(FilePrefixHistory source) { + private PrefixHistory(IoCacheViewableFileIO.PrefixHistory source) { this.source = source; } diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 09e4800975a1..261313159c68 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -22,10 +22,10 @@ import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.HadoopOptionsProvider; -import org.apache.paimon.fs.IoCacheViewableFileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.plugin.PluginLoader; @@ -203,7 +203,7 @@ public FileIO originOnly() { } @Override - public FileIO withTableView(TableFileIOView view) { + public FileIO withTableView(TableView view) { return cacheRouting == null ? this : withRouting(cacheRouting.withTableView(view)); } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java index e27183a51eb5..bba609f3aeda 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -20,9 +20,9 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.FilePrefixHistory; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; @@ -385,7 +385,7 @@ public void testTableViews() { assertThat(prefixed.targetOf(DATA_FILE).name).isEqualTo("cluster"); assertThat(prefixed.targetOf(MANIFEST).name).isEqualTo("accel"); assertThat(prefixed.withTableView(view(null, "custom-"))).isSameAs(prefixed); - assertThat(routing.withTableView(TableFileIOView.NONE)).isSameAs(routing); + assertThat(routing.withTableView(TableView.NONE)).isSameAs(routing); // Format and Object Tables send every request to the origin IoCacheRouting originOnly = prefixed.originOnly(); @@ -419,7 +419,7 @@ public void testTableViewsOnlyRouteFilesUnderTheirRoots() { // an external path on the table's bucket is one of its roots IoCacheRouting external = routing.withTableView( - TableFileIOView.of( + TableView.of( Arrays.asList(new Path(TABLE_ROOT), new Path("oss://bkt/import/")), Collections.emptyList(), null)); @@ -470,7 +470,7 @@ public void testPrefixesOfOlderSchemasAreLoadedOnceWhenOnlyTheyMayRoute() { @Test public void testFailingToLoadOlderSchemasKeepsTheirFilesOnTheOrigin() { - FilePrefixHistory failing = + PrefixHistory failing = fileIO -> { throw new IOException("schema directory not readable"); }; @@ -688,8 +688,8 @@ private static void assertType(String path, FileType type, String... changes) { .isEqualTo(type); } - private static TableFileIOView view(@Nullable FilePrefixHistory history, String... prefixes) { - return TableFileIOView.of( + private static TableView view(@Nullable PrefixHistory history, String... prefixes) { + return TableView.of( Collections.singletonList(new Path(TABLE_ROOT)), Arrays.asList(prefixes), history); } @@ -702,7 +702,7 @@ private static IoCacheRouting clone(IoCacheRouting routing) { } /** Older schemas named their data files with these prefixes; counts the loads. */ - private static class CountingHistory implements FilePrefixHistory { + private static class CountingHistory implements PrefixHistory { private static final long serialVersionUID = 1L; diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java index 68e90623ef90..b645b1fae487 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -21,10 +21,10 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.FilePrefixHistory; -import org.apache.paimon.fs.IoCacheViews; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.TableFileIOView; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.rest.RESTApi; @@ -318,8 +318,8 @@ public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); FileIO view = fileIO.originOnly(); assertThat(view).isInstanceOf(JindoFileIO.class).isNotSameAs(fileIO); - assertThat(IoCacheViews.originOnly(view)).isSameAs(view); - assertThat(IoCacheViews.withTableView(view, view("custom-"))).isSameAs(view); + assertThat(IoCacheViewableFileIO.originOnly(view)).isSameAs(view); + assertThat(IoCacheViewableFileIO.withTableView(view, view("custom-"))).isSameAs(view); view.newInputStream(DATA).close(); view.getFileStatus(MANIFEST); @@ -341,13 +341,13 @@ public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { public void testFilePrefixViewRoutesDataFilesNamedWithThem() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); Path custom = new Path(TABLE + "/dt=1/bucket-0/custom-" + UUID + "-1.orc"); - assertThat(fileIO.withTableView(TableFileIOView.NONE)).isSameAs(fileIO); + assertThat(fileIO.withTableView(TableView.NONE)).isSameAs(fileIO); fileIO.newInputStream(custom).close(); verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); FileIO view = fileIO.withTableView(view("custom-")); - assertThat(IoCacheViews.withTableView(view, view("custom-"))).isSameAs(view); + assertThat(IoCacheViewableFileIO.withTableView(view, view("custom-"))).isSameAs(view); view.newInputStream(custom).close(); view.newInputStream(DATA).close(); view.getFileStatus(MANIFEST); @@ -389,7 +389,7 @@ public void testPrefixesOfOlderSchemasAreReadFromOss() throws IOException { JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); Path older = new Path(TABLE + "/dt=1/bucket-0/older-" + UUID + "-1.orc"); List readers = new ArrayList<>(); - FilePrefixHistory history = + PrefixHistory history = reader -> { readers.add(reader); reader.exists(new Path(TABLE + "/schema")); @@ -397,7 +397,7 @@ public void testPrefixesOfOlderSchemasAreReadFromOss() throws IOException { }; FileIO view = fileIO.withTableView( - TableFileIOView.of( + TableView.of( Collections.singletonList(new Path(TABLE)), Collections.emptyList(), history)); @@ -418,7 +418,7 @@ public void testViewsUseTheFileSystemsOfTheirBase() throws Exception { Options options = cacheTargetOptions(); options.set(FILE_IO_ALLOW_CACHE, false); JindoFileIO fileIO = configuredFileIO(options); - FileIO view = IoCacheViews.originOnly(fileIO.withTableView(view("custom-"))); + FileIO view = IoCacheViewableFileIO.originOnly(fileIO.withTableView(view("custom-"))); view.newInputStream(DATA).close(); assertThat(fileIO.fsMap).hasSize(1); @@ -606,8 +606,8 @@ private static Options cacheTargetOptions() { return options; } - private static TableFileIOView view(String... prefixes) { - return TableFileIOView.of( + private static TableView view(String... prefixes) { + return TableView.of( Collections.singletonList(new Path(TABLE)), Arrays.asList(prefixes), null); }