diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java index 5091acaaff59..8728e61dfc33 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java @@ -20,9 +20,13 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.Serializable; import java.time.Duration; @@ -36,7 +40,7 @@ * An implementation of {@link FileIO} that supports multiple file system schemas. It dynamically * selects the appropriate {@link FileIO} based on the URI scheme of the given path. */ -public class ResolvingFileIO implements FileIO { +public class ResolvingFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; @@ -44,6 +48,31 @@ public class ResolvingFileIO implements FileIO { private CatalogContext context; + // a table view resolves through its base and applies the view to each FileIO + @Nullable private ResolvingFileIO base; + private TableView view = TableView.NONE; + + @Override + public FileIO originOnly() { + return withView(view.originOnly()); + } + + @Override + public FileIO withTableView(TableView tableView) { + return withView(view.merge(tableView)); + } + + private FileIO withView(TableView newView) { + if (newView == view) { + return this; + } + ResolvingFileIO result = new ResolvingFileIO(); + result.context = context; + result.base = base == null ? this : base; + result.view = newView; + return result; + } + // TODO, how to decide the real fileio is object store or not? @Override public boolean isObjectStore() { @@ -143,6 +172,10 @@ public String createBlobPresignedUrl( public FileIO fileIO(Path path) throws IOException { CacheKey cacheKey = new CacheKey(path.toUri().getScheme(), path.toUri().getAuthority()); + if (base != null) { + FileIO resolved = base.fileIO(path); + return fileIOMap.computeIfAbsent(cacheKey, k -> view.apply(resolved)); + } return fileIOMap.computeIfAbsent( cacheKey, k -> { diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java index d943abcfbcf0..12099267012d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java @@ -26,6 +26,8 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -53,7 +55,7 @@ *

After deserialization, copies of the same catalog-created wrapper lazily reuse a JVM-local * cache manager. */ -public class CachingFileIO implements FileIO { +public class CachingFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 1L; @@ -216,6 +218,28 @@ public void setRuntimeContext(Map options) { delegate.setRuntimeContext(options); } + @Override + public FileIO originOnly() { + return withDelegate(IoCacheViewableFileIO.originOnly(delegate)); + } + + @Override + public FileIO withTableView(TableView view) { + return withDelegate(IoCacheViewableFileIO.withTableView(delegate, view)); + } + + // a view shares this wrapper's cache + private FileIO withDelegate(FileIO view) { + return view == delegate + ? this + : new CachingFileIO( + view, + getOrCreateCacheManager(), + whitelist, + cacheConfiguration, + cacheNamespace); + } + @Override public void close() throws IOException { SharedCacheManager shared; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java new file mode 100644 index 000000000000..21be723de480 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fs/iocache/IoCacheViewableFileIO.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs.iocache; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.Serializable; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * A {@link FileIO} whose io-cache routing can be narrowed to the origin or to one table's files. + * Internal to io-cache routing, not a public API. + */ +public interface IoCacheViewableFileIO extends FileIO { + + /** A view of this FileIO that sends no request to an io-cache target, for mutable files. */ + FileIO originOnly(); + + /** A view of this FileIO for one table: io-cache routing only serves that table's files. */ + FileIO withTableView(TableView view); + + /** {@link #originOnly()} of a FileIO that supports views; any other FileIO as is. */ + static FileIO originOnly(FileIO fileIO) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).originOnly() + : fileIO; + } + + /** {@link #withTableView(TableView)} of a FileIO that supports views; any other as is. */ + static FileIO withTableView(FileIO fileIO, TableView view) { + return fileIO instanceof IoCacheViewableFileIO + ? ((IoCacheViewableFileIO) fileIO).withTableView(view) + : fileIO; + } + + /** Loads the data file prefixes of the schemas a table has had. */ + interface PrefixHistory extends Serializable { + + /** Reads them through {@code fileIO}, which sends no request to an io-cache target. */ + List filePrefixes(FileIO fileIO) throws IOException; + } + + /** + * What io-cache routing knows about the table a FileIO view serves: the locations it owns, the + * prefixes of its data files and where to find the prefixes of its older schemas. + */ + final class TableView implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final TableView NONE = + new TableView(false, Collections.emptyList(), Collections.emptyList(), null); + + private static final TableView ORIGIN_ONLY = + new TableView(true, Collections.emptyList(), Collections.emptyList(), null); + + private final boolean originOnly; + private final List roots; + private final List filePrefixes; + @Nullable private final PrefixHistory history; + + private TableView( + boolean originOnly, + List roots, + List filePrefixes, + @Nullable PrefixHistory history) { + this.originOnly = originOnly; + this.roots = roots; + this.filePrefixes = filePrefixes; + this.history = history; + } + + /** The view of a table whose files are under {@code roots}. */ + public static TableView of( + Collection roots, + Collection filePrefixes, + @Nullable PrefixHistory history) { + List normalizedRoots = new ArrayList<>(); + for (Path root : roots) { + addIfAbsent(normalizedRoots, normalizeRoot(root)); + } + List prefixes = new ArrayList<>(); + for (String prefix : filePrefixes) { + if (prefix != null && !prefix.trim().isEmpty()) { + addIfAbsent(prefixes, prefix); + } + } + return new TableView( + false, + Collections.unmodifiableList(normalizedRoots), + Collections.unmodifiableList(prefixes), + history); + } + + /** See {@link IoCacheViewableFileIO#originOnly()}. */ + public TableView originOnly() { + return ORIGIN_ONLY; + } + + /** This view also serving {@code other}'s table, with its history; origin only stays so. */ + public TableView merge(TableView other) { + if (originOnly || other.isNone()) { + return this; + } + if (other.originOnly || isNone()) { + return other; + } + List mergedRoots = new ArrayList<>(roots); + other.roots.forEach(root -> addIfAbsent(mergedRoots, root)); + List mergedPrefixes = new ArrayList<>(filePrefixes); + other.filePrefixes.forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); + TableView merged = + new TableView( + false, + Collections.unmodifiableList(mergedRoots), + Collections.unmodifiableList(mergedPrefixes), + other.history == null ? history : other.history); + return merged.equals(this) ? this : merged; + } + + public boolean isOriginOnly() { + return originOnly; + } + + public boolean isNone() { + return this == NONE || (!originOnly && roots.isEmpty() && filePrefixes.isEmpty()); + } + + /** Normalized {@code scheme://authority/path} of each location the table owns. */ + public List roots() { + return roots; + } + + public List filePrefixes() { + return filePrefixes; + } + + @Nullable + public PrefixHistory history() { + return history; + } + + /** Whether {@code path} is one of the roots or under one. */ + public static boolean isUnder(List roots, Path path) { + String location = normalizeRoot(path); + for (String root : roots) { + if (location.equals(root) || location.startsWith(root + "/")) { + return true; + } + } + return false; + } + + public FileIO apply(FileIO fileIO) { + if (originOnly) { + return IoCacheViewableFileIO.originOnly(fileIO); + } + return isNone() ? fileIO : IoCacheViewableFileIO.withTableView(fileIO, this); + } + + private static String normalizeRoot(Path path) { + URI uri = path.toUri(); + String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + String authority = uri.getAuthority() == null ? "" : uri.getAuthority(); + String location = uri.getPath() == null ? "" : uri.getPath(); + while (location.endsWith("/")) { + location = location.substring(0, location.length() - 1); + } + return scheme + "://" + authority + location; + } + + private static void addIfAbsent(List list, String value) { + if (!list.contains(value)) { + list.add(value); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof TableView)) { + return false; + } + TableView that = (TableView) o; + return originOnly == that.originOnly + && roots.equals(that.roots) + && filePrefixes.equals(that.filePrefixes) + && Objects.equals(history, that.history); + } + + @Override + public int hashCode() { + return Objects.hash(originOnly, roots, filePrefixes, history); + } + + @Override + public String toString() { + return originOnly + ? "TableView{originOnly}" + : "TableView{roots=" + roots + ", prefixes=" + filePrefixes + "}"; + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index 830fe058c43c..f06c55420c3d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -28,11 +28,14 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; import org.apache.paimon.options.Options; import org.apache.paimon.rest.responses.GetTableTokenResponse; import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ThreadUtils; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; @@ -44,8 +47,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -57,7 +64,7 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; /** A {@link FileIO} to support getting token from REST Server. */ -public class RESTTokenFileIO implements FileIO { +public class RESTTokenFileIO implements IoCacheViewableFileIO { private static final long serialVersionUID = 2L; @@ -67,7 +74,7 @@ public class RESTTokenFileIO implements FileIO { .defaultValue(false) .withDescription("Whether to support data token provided by the REST server."); - private static final Cache FILE_IO_CACHE = + private static final Cache FILE_IO_CACHE = Caffeine.newBuilder() .maximumSize(1000) .expireAfterAccess(10, TimeUnit.HOURS) @@ -112,12 +119,59 @@ static long fileIOCacheMaximumSize() { // Server again after serialization private volatile RESTToken token; + // applied to the FileIO of each token + private final TableView view; + private transient volatile Pair lastView; + + private transient volatile FileIOKey lastKey; + public RESTTokenFileIO( CatalogContext catalogContext, RESTApi apiInstance, Identifier identifier, Path path) { + this(catalogContext, apiInstance, identifier, path, TableView.NONE, null); + } + + private RESTTokenFileIO( + CatalogContext catalogContext, + RESTApi apiInstance, + Identifier identifier, + Path path, + TableView view, + @Nullable RESTToken token) { this.catalogContext = catalogContext; this.apiInstance = apiInstance; this.identifier = identifier; this.path = path; + this.view = view; + this.token = token; + } + + @Override + public FileIO originOnly() { + return withView(view.originOnly()); + } + + @Override + public FileIO withTableView(TableView tableView) { + return withView(view.merge(tableView)); + } + + private FileIO withView(TableView newView) { + return newView == view + ? this + : new RESTTokenFileIO( + catalogContext, apiInstance, identifier, path, newView, token); + } + + private FileIO viewOf(FileIO fileIO) { + if (view.isNone()) { + return fileIO; + } + Pair last = lastView; + if (last == null || last.getLeft() != fileIO) { + last = Pair.of(fileIO, view.apply(fileIO)); + lastView = last; + } + return last.getRight(); } @Override @@ -245,19 +299,19 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc + "REST credential lifetime after refresh."); } - FileIO fileIO = FILE_IO_CACHE.getIfPresent(currentToken); + FileIOKey key = keyOf(currentToken); + FileIO fileIO = FILE_IO_CACHE.getIfPresent(key); if (fileIO != null) { - return new FileIOWithToken(fileIO, currentToken); + return new FileIOWithToken(viewOf(fileIO), currentToken); } synchronized (FILE_IO_CACHE) { - fileIO = FILE_IO_CACHE.getIfPresent(currentToken); + fileIO = FILE_IO_CACHE.getIfPresent(key); if (fileIO != null) { - return new FileIOWithToken(fileIO, currentToken); + return new FileIOWithToken(viewOf(fileIO), currentToken); } - Options options = catalogContext.options(); - options = new Options(RESTUtil.merge(options.toMap(), currentToken.token())); + Options options = new Options(RESTUtil.merge(key.catalogOptions, currentToken.token())); options.set(FILE_IO_ALLOW_CACHE, false); CatalogContext context = CatalogContext.create( @@ -266,11 +320,25 @@ private FileIOWithToken fileIOWithToken(long minimumValidityMillis) throws IOExc catalogContext.preferIO(), catalogContext.fallbackIO()); fileIO = FileIO.get(path, context); - FILE_IO_CACHE.put(currentToken, fileIO); - return new FileIOWithToken(fileIO, currentToken); + FILE_IO_CACHE.put(key, fileIO); + return new FileIOWithToken(viewOf(fileIO), currentToken); } } + private FileIOKey keyOf(RESTToken currentToken) { + FileIOKey key = lastKey; + if (key == null || key.token != currentToken) { + Map catalogOptions = + key == null + ? Collections.unmodifiableMap( + new HashMap<>(catalogContext.options().toMap())) + : key.catalogOptions; + key = new FileIOKey(currentToken, catalogOptions); + lastKey = key; + } + return key; + } + private boolean hasRemainingLifetime(RESTToken signingToken, long minimumValidityMillis) { return signingToken == null || signingToken.expireAtMillis() - currentTimeMillis() >= minimumValidityMillis; @@ -300,6 +368,39 @@ private boolean shouldRefresh(long minimumValidityMillis) { < Math.max(TOKEN_EXPIRATION_SAFE_TIME_MILLIS, minimumValidityMillis); } + /** A cached FileIO serves one token with the catalog options it was built from. */ + private static final class FileIOKey { + + private final RESTToken token; + private final Map catalogOptions; + private final int hash; + + private FileIOKey(RESTToken token, Map catalogOptions) { + this.token = token; + this.catalogOptions = catalogOptions; + this.hash = 31 * token.hashCode() + catalogOptions.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FileIOKey)) { + return false; + } + FileIOKey that = (FileIOKey) o; + return hash == that.hash + && token.equals(that.token) + && catalogOptions.equals(that.catalogOptions); + } + + @Override + public int hashCode() { + return hash; + } + } + private static class FileIOWithToken { private final FileIO fileIO; diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java index 4d1f84cfcbb3..bfc1dcf9e0b1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/UriReaderFactory.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import javax.annotation.Nullable; @@ -96,7 +97,8 @@ protected UriReader newReader(URI uri) { try { FileIO createdFileIO = FileIO.get(new Path(uri), Objects.requireNonNull(context)); - return UriReader.fromFile(createdFileIO); + // no table owns what a descriptor points at here, and its writer may replace it + return UriReader.fromFile(IoCacheViewableFileIO.originOnly(createdFileIO)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java index 6330d68daafd..e59e46e39b1b 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.hadoop.HadoopFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; @@ -39,6 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -57,6 +59,35 @@ public void setUp() { resolvingFileIO.configure(catalogContext); } + @Test + public void testTableViewsApplyToEachResolvedFileIO() throws IOException { + IoCacheViewableFileIO oss = mock(IoCacheViewableFileIO.class); + FileIO originOnly = mock(FileIO.class); + when(oss.originOnly()).thenReturn(originOnly); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.getScheme()).thenReturn("oss"); + when(loader.load(any())).thenReturn(oss); + ResolvingFileIO fileIO = new ResolvingFileIO(); + fileIO.configure(CatalogContext.create(new Options(), loader, null)); + Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); + + FileIO view = fileIO.originOnly(); + assertInstanceOf(ResolvingFileIO.class, view); + assertSame(view, IoCacheViewableFileIO.originOnly(view)); + assertSame(originOnly, ((ResolvingFileIO) view).fileIO(data)); + assertSame(oss, fileIO.fileIO(data)); + // loading a FileIO checks access with exists + clearInvocations(oss); + view.exists(data); + verify(originOnly).exists(data); + verify(oss, never()).exists(any()); + // a FileIO without io-cache routing is its own view + Path local = new Path("file:///tmp/x"); + assertSame( + resolvingFileIO.fileIO(local), + ((ResolvingFileIO) resolvingFileIO.originOnly()).fileIO(local)); + } + @Test public void testFileIONullSchemeReturnsFallbackFileIO() throws IOException { Path path = new Path("/path/to/file"); diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index b4f3c46373dd..74bbe46dc0cf 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -28,6 +28,8 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadUtils; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.utils.FileType; @@ -43,6 +45,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -216,6 +219,40 @@ private CachingFileIO newCachingFileIO( return new CachingFileIO(delegate, cache, whitelist); } + @Test + void testTableViewsWrapTheViewOfTheDelegate() throws IOException { + IoCacheViewableFileIO delegate = mock(IoCacheViewableFileIO.class); + FileIO originOnly = mock(FileIO.class); + when(delegate.originOnly()).thenReturn(originOnly); + when(delegate.withTableView(any())).thenReturn(delegate); + CachingFileIO cachingIO = + new CachingFileIO( + delegate, + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META)); + Path data = new Path("oss://bucket/t/bucket-0/data-1.orc"); + + FileIO view = cachingIO.originOnly(); + assertThat(view).isInstanceOf(CachingFileIO.class).isNotSameAs(cachingIO); + view.exists(data); + verify(originOnly).exists(data); + verify(delegate, never()).exists(any()); + // a delegate without anything to narrow keeps the wrapper + TableView tableView = + TableView.of( + Collections.singletonList(new Path("oss://bucket/t")), + Collections.singletonList("custom-"), + null); + assertThat(cachingIO.withTableView(tableView)).isSameAs(cachingIO); + CachingFileIO plain = + new CachingFileIO( + mock(FileIO.class), + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META)); + assertThat(plain.originOnly()).isSameAs(plain); + assertThat(plain.withTableView(tableView)).isSameAs(plain); + } + @Test void testShortRemoteReadIsNotCachedAsZeroPaddedBlock() throws IOException { byte[] data = "truncated".getBytes(); diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java index e80f097fcf2e..a35297426716 100644 --- a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -26,6 +26,8 @@ import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.RemoteIterator; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; import org.apache.paimon.options.Options; import org.apache.paimon.rest.responses.GetTableTokenResponse; @@ -256,6 +258,89 @@ void testFileIOCreationFailureSurfacesAsCheckedIOException() throws IOException assertThatThrownBy(() -> fileIO.exists(tableRoot)).isInstanceOf(IOException.class); } + @Test + void testTableViewsApplyToTheFileIOOfTheToken() throws IOException { + Path tableRoot = new Path("oss://bucket/table"); + IoCacheViewableFileIO delegate = mock(IoCacheViewableFileIO.class); + FileIO originOnly = mock(FileIO.class); + FileIO prefixed = mock(FileIO.class); + TableView tableView = + TableView.of( + Collections.singletonList(tableRoot), + Collections.singletonList("custom-"), + null); + when(delegate.originOnly()).thenReturn(originOnly); + when(delegate.withTableView(tableView)).thenReturn(prefixed); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenReturn(delegate); + when(loader.getScheme()).thenReturn("oss"); + RESTApi api = mock(RESTApi.class); + Identifier identifier = Identifier.create("db", "table"); + when(api.loadTableToken(identifier)) + .thenReturn( + new GetTableTokenResponse( + Collections.singletonMap("token", UUID.randomUUID().toString()), + Long.MAX_VALUE)); + RESTTokenFileIO fileIO = + new RESTTokenFileIO( + CatalogContext.create(new Options(), loader, null), + api, + identifier, + tableRoot); + + FileIO prefixView = fileIO.withTableView(tableView); + assertThat(prefixView).isInstanceOf(RESTTokenFileIO.class).isNotSameAs(fileIO); + assertThat(IoCacheViewableFileIO.withTableView(prefixView, tableView)).isSameAs(prefixView); + assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); + assertThat(((RESTTokenFileIO) prefixView).fileIO()).isSameAs(prefixed); + // the view of the token's FileIO is built once + verify(delegate, times(1)).withTableView(any()); + + FileIO originView = fileIO.originOnly(); + assertThat(IoCacheViewableFileIO.originOnly(originView)).isSameAs(originView); + Path data = new Path("oss://bucket/table/bucket-0/data-1.orc"); + originView.exists(data); + verify(originOnly).exists(data); + verify(delegate, never()).exists(data); + assertThat(fileIO.fileIO()).isSameAs(delegate); + } + + @Test + void testFileIOsOfOneTokenAreCachedPerCatalogOptions() throws IOException { + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenAnswer(invocation -> mock(FileIO.class)); + when(loader.getScheme()).thenReturn("oss"); + Identifier identifier = Identifier.create("db", "table"); + GetTableTokenResponse response = + new GetTableTokenResponse( + Collections.singletonMap("token", UUID.randomUUID().toString()), + Long.MAX_VALUE); + + FileIO first = tokenFileIO(loader, identifier, response, "http://cache-a:80").fileIO(); + FileIO same = tokenFileIO(loader, identifier, response, "http://cache-a:80").fileIO(); + FileIO other = tokenFileIO(loader, identifier, response, "http://cache-b:80").fileIO(); + + // catalog-only options change the backend, so they scope the cached FileIO too + assertThat(same).isSameAs(first); + assertThat(other).isNotSameAs(first); + } + + private static RESTTokenFileIO tokenFileIO( + FileIOLoader loader, + Identifier identifier, + GetTableTokenResponse response, + String cacheEndpoint) { + RESTApi api = mock(RESTApi.class); + when(api.loadTableToken(identifier)).thenReturn(response); + Options options = new Options(); + options.set("io-cache.endpoint", cacheEndpoint); + return new RESTTokenFileIO( + CatalogContext.create(options, loader, null), + api, + identifier, + new Path("oss://bucket/table")); + } + @Test void testTryToWriteAtomicReachesInnerOverride() throws IOException { Path tableRoot = new Path("oss://bucket/table"); diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java index 16ea5676fb42..d42cd95833bb 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/UriReaderFactoryTest.java @@ -19,9 +19,12 @@ package org.apache.paimon.utils; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.IsolatedDirectoryFileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.utils.UriReader.FileUriReader; import org.apache.paimon.utils.UriReader.HttpUriReader; @@ -40,6 +43,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Test for {@link UriReaderFactory}. */ public class UriReaderFactoryTest { @@ -66,6 +74,24 @@ public void tearDownHttpServer() { } } + @Test + public void testFilesOfDescriptorsAreReadFromTheOriginOfTheCreatedFileIO() throws Exception { + // a descriptor may name a file another writer replaces, and no table owns it here + IoCacheViewableFileIO created = mock(IoCacheViewableFileIO.class); + FileIO originOnly = mock(FileIO.class); + when(created.originOnly()).thenReturn(originOnly); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.getScheme()).thenReturn("oss"); + when(loader.load(any())).thenReturn(created); + UriReaderFactory ossFactory = + new UriReaderFactory(CatalogContext.create(new Options(), loader, null)); + String uri = "oss://bkt/import/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1.png"; + + ossFactory.create(uri).newInputStream(uri); + verify(originOnly).newInputStream(new Path(uri)); + verify(created, never()).newInputStream(any()); + } + @Test public void testCreateHttpUriReader() { UriReader reader = factory.create("http://example.com/file.txt"); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 5a04aeb116c1..15e0d3fa7a60 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -117,7 +117,8 @@ protected AbstractFileStoreTable( Path path, TableSchema tableSchema, CatalogEnvironment catalogEnvironment) { - this.fileIO = fileIO; + // every copy, branch and time travel comes through here with its own schema + this.fileIO = TableFileIOViews.forTable(fileIO, path, tableSchema.options()); this.path = path; if (!tableSchema.options().containsKey(PATH.key())) { // make sure table is always available diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 7f3ee12d6ef2..b3521d5c6dcf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -26,6 +26,7 @@ import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; @@ -186,8 +187,9 @@ public Builder partitionManager(@Nullable FormatTablePartitionManager partitionM } public FormatTable build() { + // other writers may replace Format Table files in place return new FormatTableImpl( - fileIO, + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), identifier, rowType, partitionKeys, diff --git a/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java new file mode 100644 index 000000000000..f61386f83489 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/TableFileIOViews.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.utils.StringUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.apache.paimon.CoreOptions.CHANGELOG_FILE_PREFIX; +import static org.apache.paimon.CoreOptions.DATA_FILE_PREFIX; + +/** Builds the {@link TableView} every {@link FileStoreTable} applies to its FileIO. */ +final class TableFileIOViews { + + private TableFileIOViews() {} + + /** The FileIO of the table at {@code path} with {@code options}. */ + static FileIO forTable(FileIO fileIO, Path path, Map options) { + List prefixes = new ArrayList<>(2); + prefixes.add(options.get(DATA_FILE_PREFIX.key())); + prefixes.add(options.get(CHANGELOG_FILE_PREFIX.key())); + // only files under the table's location use io-cache, not those under external paths + return IoCacheViewableFileIO.withTableView( + fileIO, + TableView.of( + Collections.singletonList(path), + prefixes, + new SchemaFilePrefixHistory(path, CoreOptions.branch(options)))); + } + + /** The data and changelog file prefixes of every schema of a branch, read from storage. */ + static final class SchemaFilePrefixHistory implements PrefixHistory { + + private static final long serialVersionUID = 1L; + + private final Path path; + private final String branch; + + SchemaFilePrefixHistory(Path path, String branch) { + this.path = path; + this.branch = branch; + } + + @Override + public List filePrefixes(FileIO fileIO) { + List prefixes = new ArrayList<>(); + for (TableSchema schema : new FileSystemSchemaManager(fileIO, path, branch).listAll()) { + for (String key : + new String[] {DATA_FILE_PREFIX.key(), CHANGELOG_FILE_PREFIX.key()}) { + String prefix = schema.options().get(key); + if (!StringUtils.isNullOrWhitespaceOnly(prefix) && !prefixes.contains(prefix)) { + prefixes.add(prefix); + } + } + } + return prefixes; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SchemaFilePrefixHistory)) { + return false; + } + SchemaFilePrefixHistory that = (SchemaFilePrefixHistory) o; + return path.equals(that.path) && branch.equals(that.branch); + } + + @Override + public int hashCode() { + return Objects.hash(path, branch); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java index f8c67fc77681..0cbbb0fe3fd3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/iceberg/IcebergTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -93,8 +94,16 @@ public Builder uuid(String uuid) { } public IcebergTable build() { + // other engines write Iceberg tables, so their files never use io-cache return new IcebergTableImpl( - identifier, fileIO, rowType, partitionKeys, location, options, comment, uuid); + identifier, + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), + rowType, + partitionKeys, + location, + options, + comment, + uuid); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java index 93146a41a814..d46ff7583406 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/lance/LanceTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.RowType; @@ -82,7 +83,14 @@ public Builder comment(String comment) { } public LanceTable build() { - return new LanceTableImpl(identifier, fileIO, rowType, location, options, comment); + // other engines write Lance datasets, so their files never use io-cache + return new LanceTableImpl( + identifier, + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), + rowType, + location, + options, + comment); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java index 98c95d6d8ed8..ee123bedaa23 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -91,7 +92,13 @@ public ObjectTable.Builder comment(String comment) { } public ObjectTable build() { - return new ObjectTableImpl(identifier, fileIO, location, options, comment); + // other writers manage the objects, so they may change under the same name + return new ObjectTableImpl( + identifier, + fileIO == null ? null : IoCacheViewableFileIO.originOnly(fileIO), + location, + options, + comment); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java new file mode 100644 index 000000000000..ca17df122691 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/IoCacheTableRoutingTest.java @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.UriReader; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Reads through an io-cache target that keeps the first copy it served while origin moves on, as + * the review of apache/paimon-rust#1014 does, through the catalog API and real writers. + */ +class IoCacheTableRoutingTest { + + private static final byte[] OLD = "OLD".getBytes(StandardCharsets.UTF_8); + private static final byte[] NEW = "NEW".getBytes(StandardCharsets.UTF_8); + + @TempDir java.nio.file.Path tempDir; + + private Target target; + private FileSystemCatalog catalog; + + @BeforeEach + void before() throws Exception { + target = new Target(); + catalog = + new FileSystemCatalog( + new TargetRetainingFileIO(target, false, TableView.NONE), + new Path(tempDir.resolve("warehouse").toUri().toString())); + catalog.createDatabase("db", true); + } + + @ParameterizedTest(name = "primary key {0}") + @ValueSource(booleans = {true, false}) + void testDescriptorFieldsOutsideTheTableAreReadFromTheOrigin(boolean primaryKey) + throws Exception { + Schema.Builder schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.BLOB()) + .option(CoreOptions.BLOB_DESCRIPTOR_FIELD.key(), "payload"); + if (primaryKey) { + schema.primaryKey("id").option(CoreOptions.BUCKET.key(), "1"); + } else { + schema.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_FIELD.key(), "payload"); + } + FileStoreTable table = createTable("descriptors", schema.build()); + Path source = replacedSource(); + + write(table, GenericRow.of(1, descriptorOf(LocalFileIO.create(), source))); + + List rows = read(table); + assertThat(rows.get(0).getBlob(1).toData()).isEqualTo(NEW); + assertThat(target.reads).doesNotContain(source.getName()); + // the data file the table wrote still uses the target + assertThat(target.reads).anyMatch(name -> name.startsWith("data-")); + } + + @Test + void testBlobSourcesOutsideTheTableAreCopiedFromTheOrigin() throws Exception { + FileStoreTable table = + createTable( + "managed", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.BLOB()) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_FIELD.key(), "payload") + .build()); + Path source = replacedSource(); + + // the writer reads the source through the table's FileIO and copies it into a .blob file + write(table, GenericRow.of(1, descriptorOf(table.fileIO(), source))); + + assertThat(target.reads).doesNotContain(source.getName()); + assertThat(read(table).get(0).getBlob(1).toData()).isEqualTo(NEW); + } + + @Test + void testDataFilesUnderExternalPathsAreReadFromTheOrigin() throws Exception { + String external = tempDir.resolve("external").toUri().toString(); + FileStoreTable table = + createTable( + "external", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), external) + .option( + CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(), + "round-robin") + .build()); + write(table, GenericRow.of(1)); + List externalFiles = new ArrayList<>(); + try (Stream files = Files.walk(tempDir.resolve("external"))) { + files.filter(Files::isRegularFile).forEach(externalFiles::add); + } + assertThat(externalFiles) + .anyMatch(file -> file.getFileName().toString().startsWith("data-")); + + target.reads.clear(); + assertThat(read(table)).hasSize(1); + for (java.nio.file.Path file : externalFiles) { + assertThat(target.reads).doesNotContain(file.getFileName().toString()); + } + } + + @Test + void testFilesOfOlderSchemaPrefixesUseTheTargetAfterAReload() throws Exception { + FileStoreTable table = + createTable( + "altered", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "older-") + .build()); + write(table, GenericRow.of(1)); + new FileSystemSchemaManager(table.fileIO(), table.location()) + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "newer-")); + write(table(table.name()), GenericRow.of(2)); + + target.reads.clear(); + // a fresh catalog knows only the newest schema until a file needs the older ones + before(); + assertThat(read(table(table.name()))).hasSize(2); + assertThat(target.reads).anyMatch(name -> name.startsWith("older-")); + assertThat(target.reads).anyMatch(name -> name.startsWith("newer-")); + assertThat(target.historyLoads).hasSize(1); + } + + @Test + void testBranchFilesUseTheTargetWithTheBranchPrefix() throws Exception { + FileStoreTable table = + createTable( + "branched", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "main-") + .build()); + table.createBranch("dev"); + new FileSystemSchemaManager(table.fileIO(), table.location(), "dev") + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "branch-")); + Identifier dev = new Identifier("db", table.name(), "dev"); + write((FileStoreTable) catalog.getTable(dev), GenericRow.of(1)); + + for (FileStoreTable branch : + new FileStoreTable[] { + (FileStoreTable) catalog.getTable(dev), + table(table.name()).switchToBranch("dev") + }) { + target.reads.clear(); + assertThat(read(branch)).hasSize(1); + assertThat(target.reads).anyMatch(name -> name.startsWith("branch-")); + } + } + + @Test + void testBranchFilesOfOlderMainPrefixesUseTheTarget() throws Exception { + FileStoreTable table = + createTable( + "inherited", + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "older-") + .build()); + write(table, GenericRow.of(1)); + new FileSystemSchemaManager(table.fileIO(), table.location()) + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "newer-")); + write(table(table.name()), GenericRow.of(2)); + table(table.name()).createTag("tag"); + table(table.name()).createBranch("dev", "tag"); + + // the branch keeps copies of the schemas its files were written with + before(); + Identifier dev = new Identifier("db", table.name(), "dev"); + target.reads.clear(); + assertThat(read((FileStoreTable) catalog.getTable(dev))).hasSize(2); + assertThat(target.reads).anyMatch(name -> name.startsWith("older-")); + assertThat(target.reads).anyMatch(name -> name.startsWith("newer-")); + } + + private FileStoreTable createTable(String name, Schema schema) throws Exception { + catalog.createTable(Identifier.create("db", name), schema, false); + return table(name); + } + + private FileStoreTable table(String name) throws Exception { + return (FileStoreTable) catalog.getTable(Identifier.create("db", name)); + } + + // Named like a data file and outside the table; the target keeps OLD, origin has NEW. + private Path replacedSource() throws IOException { + Path source = + new Path( + tempDir.resolve("import") + .resolve("data-" + UUID.randomUUID() + "-1.png") + .toUri() + .toString()); + LocalFileIO.create().writeFile(source, new String(NEW, StandardCharsets.UTF_8), false); + target.retained.put(source.toString(), OLD); + return source; + } + + private static Blob descriptorOf(FileIO fileIO, Path source) { + return Blob.fromDescriptor( + UriReader.fromFile(fileIO), new BlobDescriptor(source.toString(), 0, NEW.length)); + } + + private static void write(FileStoreTable table, InternalRow row) throws Exception { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(row); + commit.commit(write.prepareCommit()); + } + } + + private static List read(FileStoreTable table) throws Exception { + ReadBuilder builder = table.newReadBuilder(); + List rows = new ArrayList<>(); + builder.newRead().createReader(builder.newScan().plan()).forEachRemaining(rows::add); + return rows; + } + + /** What the io-cache target serves and keeps, shared by every view. */ + private static class Target implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Map retained = new ConcurrentHashMap<>(); + private final List reads = new CopyOnWriteArrayList<>(); + private final List historyLoads = new CopyOnWriteArrayList<>(); + } + + /** + * Local files whose reads of Paimon data files go to a target that keeps its first copy, routed + * with the table view the way JindoFileIO routes them. + */ + private static class TargetRetainingFileIO extends LocalFileIO + implements IoCacheViewableFileIO { + + private static final long serialVersionUID = 1L; + + private static final String SUFFIX = + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + + "(-[0-9]+\\..+|\\.managed\\.blob)"; + private static final Pattern DATA_FILE_SUFFIX = Pattern.compile(SUFFIX); + private static final Pattern ANY_PREFIX_DATA_FILE = Pattern.compile(".*" + SUFFIX); + + private final Target target; + private final boolean originOnly; + private final TableView view; + private transient volatile List olderPrefixes; + + private TargetRetainingFileIO(Target target, boolean originOnly, TableView view) { + this.target = target; + this.originOnly = originOnly; + this.view = view; + } + + @Override + public FileIO originOnly() { + return originOnly ? this : new TargetRetainingFileIO(target, true, view); + } + + @Override + public FileIO withTableView(TableView tableView) { + TableView merged = view.merge(tableView); + return originOnly || merged.equals(view) + ? this + : new TargetRetainingFileIO(target, false, merged); + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + if (!routes(path)) { + return super.newInputStream(path); + } + target.reads.add(path.getName()); + byte[] bytes = target.retained.get(path.toString()); + if (bytes == null) { + bytes = readAll(path); + target.retained.put(path.toString(), bytes); + } + return new ByteArraySeekableStream(bytes); + } + + private byte[] readAll(Path path) throws IOException { + try (SeekableInputStream in = super.newInputStream(path)) { + byte[] bytes = new byte[(int) getFileStatus(path).getLen()]; + int read = 0; + while (read < bytes.length) { + int n = in.read(bytes, read, bytes.length - read); + if (n < 0) { + break; + } + read += n; + } + return bytes; + } + } + + private boolean routes(Path path) throws IOException { + if (originOnly || (!view.roots().isEmpty() && !TableView.isUnder(view.roots(), path))) { + return false; + } + List prefixes = new ArrayList<>(view.filePrefixes()); + prefixes.add("data-"); + prefixes.add("changelog-"); + if (isDataFile(path.getName(), prefixes)) { + return true; + } + if (view.history() == null || !ANY_PREFIX_DATA_FILE.matcher(path.getName()).matches()) { + return false; + } + if (olderPrefixes == null) { + target.historyLoads.add(path.getName()); + olderPrefixes = view.history().filePrefixes(originOnly()); + } + return isDataFile(path.getName(), olderPrefixes); + } + + private static boolean isDataFile(String name, List prefixes) { + for (String prefix : prefixes) { + if (name.startsWith(prefix) + && DATA_FILE_SUFFIX.matcher(name.substring(prefix.length())).matches()) { + return true; + } + } + return false; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java new file mode 100644 index 000000000000..6bd12bdc1347 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/TableFileIOViewTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.iceberg.IcebergTable; +import org.apache.paimon.table.lance.LanceTable; +import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests that each table takes its io-cache view of the catalog FileIO. */ +class TableFileIOViewTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + void testTablesTakeTheirViewOfTheCatalogFileIO() throws Exception { + ViewFileIO fileIO = new ViewFileIO(false, TableView.NONE); + try (FileSystemCatalog catalog = + new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + // this catalog only loads Format Tables, so write the schema directly + Identifier format = Identifier.create("db", "format"); + new FileSystemSchemaManager(fileIO, catalog.getTableLocation(format)) + .createTable( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option( + CoreOptions.TYPE.key(), + TableType.FORMAT_TABLE.toString()) + .build()); + Identifier custom = Identifier.create("db", "custom"); + catalog.createTable( + custom, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "custom-") + .option(CoreOptions.CHANGELOG_FILE_PREFIX.key(), "cl-") + .build(), + false); + Identifier plain = Identifier.create("db", "plain"); + catalog.createTable( + plain, Schema.newBuilder().column("id", DataTypes.INT()).build(), false); + + // other writers may replace Format Table files in place + Table formatTable = catalog.getTable(format); + assertThat(viewOf(formatTable).originOnly).isTrue(); + assertThat(formatTable.copy(Collections.emptyMap()).fileIO()) + .isSameAs(formatTable.fileIO()); + + // a Paimon table owns its location and passes the file prefixes of its schema + FileStoreTable table = (FileStoreTable) catalog.getTable(custom); + TableView view = viewOf(table).view; + assertThat(viewOf(table).originOnly).isFalse(); + assertThat(view.roots()).containsExactly(root(table.location())); + assertThat(view.filePrefixes()).containsExactly("custom-", "cl-"); + assertThat(view.history()) + .isEqualTo( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "main")); + FileStoreTable copy = + table.copy( + Collections.singletonMap( + CoreOptions.DATA_FILE_PREFIX.key(), "dynamic-")); + assertThat(viewOf(copy).view.filePrefixes()) + .containsExactly("custom-", "cl-", "dynamic-"); + assertThat(table.copy(Collections.emptyMap()).fileIO()).isSameAs(table.fileIO()); + assertThat(table.copyWithLatestSchema().fileIO()).isSameAs(table.fileIO()); + + FileStoreTable plainTable = (FileStoreTable) catalog.getTable(plain); + assertThat(viewOf(plainTable).view.roots()) + .containsExactly(root(plainTable.location())); + assertThat(viewOf(plainTable).view.filePrefixes()).isEmpty(); + } + + ObjectTable objectTable = + ObjectTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "objects")) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(objectTable).originOnly).isTrue(); + IcebergTable icebergTable = + IcebergTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "iceberg")) + .rowType(RowType.of(DataTypes.INT())) + .partitionKeys(Collections.emptyList()) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(icebergTable).originOnly).isTrue(); + LanceTable lanceTable = + LanceTable.builder() + .fileIO(fileIO) + .identifier(Identifier.create("db", "lance")) + .rowType(RowType.of(DataTypes.INT())) + .location(tempDir.toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(viewOf(lanceTable).originOnly).isTrue(); + } + + @Test + void testBranchesAndExternalPathsTakeTheirOwnView() throws Exception { + ViewFileIO fileIO = new ViewFileIO(false, TableView.NONE); + String external = tempDir.resolve("external").toUri().toString(); + try (FileSystemCatalog catalog = + new FileSystemCatalog(fileIO, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + Identifier identifier = Identifier.create("db", "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.DATA_FILE_PREFIX.key(), "main-") + .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), external) + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + // only the table's own location: its external paths read from origin + assertThat(viewOf(table).view.roots()).containsExactly(root(table.location())); + + table.createBranch("dev"); + FileStoreTable dev = table.switchToBranch("dev"); + assertThat(viewOf(dev).view.history()) + .isEqualTo( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "dev")); + new FileSystemSchemaManager(fileIO, table.location(), "dev") + .commitChanges( + SchemaChange.setOption(CoreOptions.DATA_FILE_PREFIX.key(), "branch-")); + FileStoreTable reloaded = + (FileStoreTable) catalog.getTable(new Identifier("db", "t", "dev")); + assertThat(viewOf(reloaded).view.filePrefixes()).containsExactly("branch-"); + assertThat(viewOf(table.switchToBranch("dev")).view.filePrefixes()).contains("branch-"); + + // the history of a branch lists its own schemas, from storage + assertThat( + new TableFileIOViews.SchemaFilePrefixHistory(table.location(), "dev") + .filePrefixes(LocalFileIO.create())) + .containsExactlyInAnyOrder("main-", "branch-"); + } + } + + @Test + void testFileIOsWithoutViewsStayTheSameInstance() throws Exception { + // a decorator that forwards every FileIO method, default ones included + LocalFileIO local = LocalFileIO.create(); + FileIO forwarding = + (FileIO) + Proxy.newProxyInstance( + FileIO.class.getClassLoader(), + new Class[] {FileIO.class}, + (proxy, method, args) -> { + try { + return method.invoke(local, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + FormatTable formatTable = + FormatTable.builder() + .fileIO(forwarding) + .identifier(Identifier.create("db", "format")) + .rowType(RowType.of(DataTypes.INT())) + .partitionKeys(Collections.emptyList()) + .location(tempDir.resolve("format").toUri().toString()) + .format(FormatTable.Format.PARQUET) + .options(Collections.emptyMap()) + .build(); + assertThat(formatTable.fileIO()).isSameAs(forwarding); + ObjectTable objectTable = + ObjectTable.builder() + .fileIO(forwarding) + .identifier(Identifier.create("db", "objects")) + .location(tempDir.resolve("objects").toUri().toString()) + .options(Collections.emptyMap()) + .build(); + assertThat(objectTable.fileIO()).isSameAs(forwarding); + try (FileSystemCatalog catalog = + new FileSystemCatalog(forwarding, new Path(tempDir.toUri().toString()))) { + catalog.createDatabase("db", false); + Identifier identifier = Identifier.create("db", "t"); + catalog.createTable( + identifier, Schema.newBuilder().column("id", DataTypes.INT()).build(), false); + assertThat(catalog.getTable(identifier).fileIO()).isSameAs(forwarding); + } + } + + private static ViewFileIO viewOf(Table table) { + return (ViewFileIO) table.fileIO(); + } + + private static String root(Path path) { + return TableView.of(Collections.singletonList(path), Collections.emptyList(), null) + .roots() + .get(0); + } + + /** A FileIO that records the views tables take. */ + private static class ViewFileIO extends LocalFileIO implements IoCacheViewableFileIO { + + private static final long serialVersionUID = 1L; + + private final boolean originOnly; + private final TableView view; + + private ViewFileIO(boolean originOnly, TableView view) { + this.originOnly = originOnly; + this.view = view; + } + + @Override + public FileIO originOnly() { + return originOnly ? this : new ViewFileIO(true, view); + } + + @Override + public FileIO withTableView(TableView tableView) { + TableView merged = view.merge(tableView); + return originOnly || merged.equals(view) ? this : new ViewFileIO(false, merged); + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java index 64fc3e07c708..71e51347401d 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/HadoopCompliantFileIO.java @@ -26,6 +26,8 @@ import org.apache.paimon.fs.RemoteIterator; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.options.Options; import org.apache.paimon.utils.Pair; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -37,6 +39,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -58,6 +62,7 @@ public abstract class HadoopCompliantFileIO implements FileIO { private static final Logger LOG = LoggerFactory.getLogger(HadoopCompliantFileIO.class); private static final long serialVersionUID = 1L; + private static final String JINDO_CACHE_RPC_ADDRESS = "fs.jindocache.namespace.rpc.address"; /// Detailed cache strategies are retrieved from REST server. private static final String META_CACHE_ENABLED_TAG = "meta"; @@ -68,14 +73,21 @@ public abstract class HadoopCompliantFileIO implements FileIO { protected boolean metaCacheEnabled = false; protected boolean readCacheEnabled = false; protected boolean writeCacheEnabled = false; + protected boolean existsCacheEnabled = false; protected transient volatile Map> fsMap; protected transient volatile Map> jindoCacheFsMap; + // Non-null when endpoint routing is configured instead of JindoCache RPC. + @Nullable IoCacheRouting cacheRouting; + // Only enable cache for path which is generated with uuid private List cacheWhitelistPaths = new ArrayList<>(); boolean shouldCache(Path path) { + if (cacheRouting != null) { + return cacheTargetOf(path) != null; + } if (cacheWhitelistPaths.isEmpty()) { return true; } @@ -90,28 +102,44 @@ boolean shouldCache(Path path) { @Override public void configure(CatalogContext context) { - // Process file io cache configuration - if (!context.options().get(IO_CACHE_ENABLED) - || context.options().get(IO_CACHE_POLICY) == null - || context.options().get(IO_CACHE_POLICY).contains(DISABLE_CACHE_TAG)) { + Options options = context.options(); + // Configure cache for the selected backend. + if (!options.get(IO_CACHE_ENABLED) || options.get(IO_CACHE_POLICY) == null) { LOG.debug( "Cache is disabled with io-cache.enabled={}, io-cache.policy={}", - context.options().get(IO_CACHE_ENABLED), - context.options().get(IO_CACHE_POLICY)); + options.get(IO_CACHE_ENABLED), + options.get(IO_CACHE_POLICY)); return; } - // Enable file io cache - if (context.options().get("fs.jindocache.namespace.rpc.address") == null) { + if (options.get(JINDO_CACHE_RPC_ADDRESS) == null) { + cacheRouting = IoCacheRouting.create(options); + if (cacheRouting == null) { + LOG.debug( + "FileIO cache endpoint routing is not configured for io-cache.policy={}", + options.get(IO_CACHE_POLICY)); + return; + } + metaCacheEnabled = cacheRouting.metaCacheEnabled(); + readCacheEnabled = cacheRouting.readCacheEnabled(); + writeCacheEnabled = cacheRouting.writeCacheEnabled(); + existsCacheEnabled = cacheRouting.existsCacheEnabled(); LOG.info( - "FileIO cache is enabled but JindoCache RPC address is not set, fallback to no-cache"); + "Cache endpoints enabled: meta {}, read {}, write {}, exists {}, {}", + metaCacheEnabled, + readCacheEnabled, + writeCacheEnabled, + existsCacheEnabled, + cacheRouting); } else { - metaCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(META_CACHE_ENABLED_TAG); - readCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(READ_CACHE_ENABLED_TAG); - writeCacheEnabled = - context.options().get(IO_CACHE_POLICY).contains(WRITE_CACHE_ENABLED_TAG); - String whitelist = context.options().get(IO_CACHE_WHITELIST_PATH); + // Keep legacy JindoCache policy matching unchanged; endpoint routing uses exact tokens. + if (options.get(IO_CACHE_POLICY).contains(DISABLE_CACHE_TAG)) { + return; + } + metaCacheEnabled = options.get(IO_CACHE_POLICY).contains(META_CACHE_ENABLED_TAG); + readCacheEnabled = options.get(IO_CACHE_POLICY).contains(READ_CACHE_ENABLED_TAG); + writeCacheEnabled = options.get(IO_CACHE_POLICY).contains(WRITE_CACHE_ENABLED_TAG); + existsCacheEnabled = metaCacheEnabled; + String whitelist = options.get(IO_CACHE_WHITELIST_PATH); if (!whitelist.equals("*")) { cacheWhitelistPaths = Lists.newArrayList(whitelist.split(",")); } @@ -194,7 +222,7 @@ public FileStatus next() throws IOException { @Override public boolean exists(Path path) throws IOException { org.apache.hadoop.fs.Path hadoopPath = path(path); - boolean shouldCache = metaCacheEnabled && shouldCache(path); + boolean shouldCache = existsCacheEnabled && shouldCache(path); LOG.debug("Exists should cache {} for path {}", shouldCache, path); return getFileSystem(hadoopPath, shouldCache).exists(hadoopPath); } @@ -226,6 +254,19 @@ public boolean rename(Path src, Path dst) throws IOException { return getFileSystem(hadoopSrc, false).rename(hadoopSrc, hadoopDst); } + /** + * The cache target of a file, loading the prefixes of older schemas if only they may route it. + */ + @Nullable + IoCacheRouting.CacheTarget cacheTargetOf(Path path) { + IoCacheRouting.CacheTarget target = cacheRouting.targetOf(path); + if (target == null && cacheRouting.awaitsHistory(path)) { + cacheRouting.loadHistory(IoCacheViewableFileIO.originOnly(this)); + target = cacheRouting.targetOf(path); + } + return target; + } + protected org.apache.hadoop.fs.Path path(Path path) { URI uri = path.toUri(); if ("oss".equals(uri.getScheme()) && uri.getUserInfo() != null) { diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java new file mode 100644 index 000000000000..b3d9f3c13ef0 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/IoCacheRouting.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.FileType; +import org.apache.paimon.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.apache.paimon.CoreOptions.CHANGELOG_FILE_PREFIX; +import static org.apache.paimon.CoreOptions.DATA_FILE_PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.DLF_OSS_ENDPOINT; +import static org.apache.paimon.rest.RESTCatalogOptions.IO_CACHE_ENABLED; +import static org.apache.paimon.rest.RESTCatalogOptions.IO_CACHE_POLICY; + +/** + * Selects cache targets for immutable Paimon files using {@code io-cache.targets}, {@code + * io-cache.routes} and {@code io-cache.whitelist}. The endpoint mode also carries the normalized + * {@code io-cache.policy} read, metadata, existence and write switches. + */ +final class IoCacheRouting implements Serializable { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(IoCacheRouting.class); + + private static final String META_CACHE_ENABLED_TAG = "meta"; + private static final String READ_CACHE_ENABLED_TAG = "read"; + private static final String WRITE_CACHE_ENABLED_TAG = "write"; + // exists asks whether a file is still there, which a cache of deleted files can get wrong + private static final String EXISTS_CACHE_ENABLED_TAG = "exists"; + private static final String DISABLE_CACHE_TAG = "none"; + + private static final Pattern TARGET_NAME = Pattern.compile("[a-z][a-z0-9-]*"); + + // What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone + // writer's -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack {uuid}.managed.blob. + private static final String UUID = + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; + private static final Pattern DATA_FILE_SUFFIX = + Pattern.compile( + "(?:-u-.+-s-[0-9]+-w-)?" + UUID + "-[0-9]+\\..+|" + UUID + "\\.managed\\.blob"); + // The name of a data file or its file index, whatever its prefix. + private static final Pattern ANY_PREFIX_DATA_FILE = + Pattern.compile(".*(?:" + UUID + "-[0-9]+\\..+|" + UUID + "\\.managed\\.blob)"); + + // Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to + // changelog manifests); Format Table files may be replaced in place. + private static final Pattern META_FILE = + Pattern.compile( + "(manifest-list|index-manifest|stat)-" + + UUID + + "-[0-9]+|manifest-" + + UUID + + "(-changelog)?-[0-9]+(\\.avro\\.sidecar)?"); + private static final Pattern BUCKET_INDEX_FILE = Pattern.compile("index-" + UUID + "-[0-9]+"); + private static final Pattern GLOBAL_INDEX_FILE = + Pattern.compile("[a-z0-9_-]+-global-index-" + UUID + "\\.index"); + + private static final String MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar"; + + @Nullable private final String ossEndpoint; + private final List targets; + private final Map targetByType; + private final List dataFilePrefixes; + // Locations of the table a view serves; empty when this routing serves any table. + private final List roots; + @Nullable private final PrefixHistory history; + private final boolean metaCacheEnabled; + private final boolean readCacheEnabled; + private final boolean writeCacheEnabled; + private final boolean existsCacheEnabled; + + private IoCacheRouting( + @Nullable String ossEndpoint, + List targets, + Map targetByType, + List dataFilePrefixes, + List roots, + @Nullable PrefixHistory history, + boolean metaCacheEnabled, + boolean readCacheEnabled, + boolean writeCacheEnabled, + boolean existsCacheEnabled) { + this.ossEndpoint = ossEndpoint; + this.targets = targets; + this.targetByType = targetByType; + this.dataFilePrefixes = dataFilePrefixes; + this.roots = roots; + this.history = history; + this.metaCacheEnabled = metaCacheEnabled; + this.readCacheEnabled = readCacheEnabled; + this.writeCacheEnabled = writeCacheEnabled; + this.existsCacheEnabled = existsCacheEnabled; + } + + /** Creates endpoint routing from the options, or null when routing is not enabled. */ + @Nullable + static IoCacheRouting create(Options options) { + if (!options.get(IO_CACHE_ENABLED)) { + return null; + } + Set policy = parsePolicy(options.get(IO_CACHE_POLICY)); + boolean metaCache = policy.contains(META_CACHE_ENABLED_TAG); + boolean readCache = policy.contains(READ_CACHE_ENABLED_TAG); + boolean writeCache = policy.contains(WRITE_CACHE_ENABLED_TAG); + boolean existsCache = policy.contains(EXISTS_CACHE_ENABLED_TAG); + if (policy.contains(DISABLE_CACHE_TAG) + || !(metaCache || readCache || writeCache || existsCache)) { + return null; + } + // a client-side dlf.oss-endpoint replaces every endpoint the token vends + if (!StringUtils.isNullOrWhitespaceOnly(options.get(DLF_OSS_ENDPOINT.key()))) { + return null; + } + Map urls = declaredTargets(options); + if (urls.isEmpty()) { + return null; + } + Map byName = new LinkedHashMap<>(); + urls.forEach( + (name, url) -> { + if (!StringUtils.isNullOrWhitespaceOnly(url)) { + byName.put(name, target(options, name, url.trim())); + } + }); + + String whitelist = options.get("io-cache.whitelist"); + Set types = + whitelist == null ? EnumSet.allOf(FileType.class) : fileTypes(whitelist); + String routes = options.get("io-cache.routes"); + Map targetByType = new EnumMap<>(FileType.class); + for (FileType type : types) { + CacheTarget target = + routes == null + ? byName.values().stream().findFirst().orElse(null) + : byName.get(routedName(routes, type)); + if (target != null) { + targetByType.put(type, target); + } + } + + String origin = options.get("io-cache.origin.endpoint"); + return new IoCacheRouting( + StringUtils.isNullOrWhitespaceOnly(origin) + ? options.get("fs.oss.endpoint") + : origin, + new ArrayList<>(byName.values()), + targetByType, + dataFilePrefixes(options), + Collections.emptyList(), + null, + metaCache, + readCache, + writeCache, + existsCache); + } + + /** This routing with no request on a cache target. */ + IoCacheRouting originOnly() { + return targetByType.isEmpty() + ? this + : new IoCacheRouting( + ossEndpoint, + targets, + new EnumMap<>(FileType.class), + dataFilePrefixes, + roots, + null, + metaCacheEnabled, + readCacheEnabled, + writeCacheEnabled, + existsCacheEnabled); + } + + /** + * This routing for the table of {@code view}: only files under its roots may use a cache + * target, also when named with its prefixes or those of its older schemas. + */ + IoCacheRouting withTableView(TableView view) { + if (view.isOriginOnly()) { + return originOnly(); + } + if (targetByType.isEmpty() || view.isNone()) { + return this; + } + List mergedPrefixes = new ArrayList<>(dataFilePrefixes); + view.filePrefixes().forEach(prefix -> addIfAbsent(mergedPrefixes, prefix)); + List mergedRoots = new ArrayList<>(roots); + view.roots().forEach(root -> addIfAbsent(mergedRoots, root)); + PrefixHistory mergedHistory = + view.history() == null + ? history + : history != null && history.source.equals(view.history()) + ? history + : new PrefixHistory(view.history()); + if (mergedPrefixes.equals(dataFilePrefixes) + && mergedRoots.equals(roots) + && mergedHistory == history) { + return this; + } + return new IoCacheRouting( + ossEndpoint, + targets, + targetByType, + mergedPrefixes, + mergedRoots, + mergedHistory, + metaCacheEnabled, + readCacheEnabled, + writeCacheEnabled, + existsCacheEnabled); + } + + private static Set parsePolicy(@Nullable String value) { + return value == null + ? Collections.emptySet() + : Arrays.stream(value.split(",")) + .map(token -> token.trim().toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + } + + boolean metaCacheEnabled() { + return metaCacheEnabled; + } + + boolean readCacheEnabled() { + return readCacheEnabled; + } + + boolean writeCacheEnabled() { + return writeCacheEnabled; + } + + boolean existsCacheEnabled() { + return existsCacheEnabled; + } + + /** The OSS endpoint used by requests that do not use a cache target. */ + @Nullable + String ossEndpoint() { + return ossEndpoint; + } + + Collection targets() { + return targets; + } + + /** The cache target selected for a file, or null when it does not use a cache target. */ + @Nullable + CacheTarget targetOf(Path path) { + // Only OSS paths can use cache targets, and a table view only its own files. + if (!"oss".equals(path.toUri().getScheme()) + || (!roots.isEmpty() && !TableView.isUnder(roots, path))) { + return null; + } + FileType type = cacheableType(path, filePrefixes()); + return type == null ? null : targetByType.get(type); + } + + /** Whether only the prefixes of older schemas, not loaded yet, may route {@code path}. */ + boolean awaitsHistory(Path path) { + if (history == null + || history.loaded != null + || !(targetByType.containsKey(FileType.DATA) + || targetByType.containsKey(FileType.FILE_INDEX)) + || (!roots.isEmpty() && !TableView.isUnder(roots, path))) { + return false; + } + String name = path.getName(); + return !name.endsWith(MANIFEST_SIDECAR_SUFFIX) + && ANY_PREFIX_DATA_FILE.matcher(name).matches() + && !isDataFileName(name, dataFilePrefixes); + } + + /** Loads the prefixes of older schemas once, through a FileIO that only uses OSS. */ + void loadHistory(FileIO originOnly) { + if (history != null) { + history.load(originOnly); + } + } + + private List filePrefixes() { + List loaded = history == null ? null : history.loaded; + if (loaded == null || loaded.isEmpty()) { + return dataFilePrefixes; + } + List prefixes = new ArrayList<>(dataFilePrefixes); + loaded.forEach(prefix -> addIfAbsent(prefixes, prefix)); + return prefixes; + } + + private static void addIfAbsent(List list, String value) { + if (!StringUtils.isNullOrWhitespaceOnly(value) && !list.contains(value)) { + list.add(value); + } + } + + /** The type of a file a cache may serve, or null when it must be read from OSS. */ + @Nullable + static FileType cacheableType(Path path, List dataFilePrefixes) { + String name = path.getName(); + if (isTemporary(name) || FileType.isMutable(path)) { + return null; + } + if (isDataFileName(name, dataFilePrefixes)) { + return name.endsWith(".index") ? FileType.FILE_INDEX : FileType.DATA; + } + if (isSequential(path)) { + return null; + } + FileType type = FileType.classify(path); + return isPaimonFileName(type, name) ? type : null; + } + + private static boolean isPaimonFileName(FileType type, String name) { + switch (type) { + case META: + return META_FILE.matcher(name).matches(); + case BUCKET_INDEX: + return BUCKET_INDEX_FILE.matcher(name).matches(); + case GLOBAL_INDEX: + return GLOBAL_INDEX_FILE.matcher(name).matches(); + default: + // data files and their file indexes are recognized by isDataFileName + return false; + } + } + + // Manifests, indexes and statistics share the uuid-count shape but have no extension. + private static boolean isDataFileName(String name, List dataFilePrefixes) { + if (name.endsWith(MANIFEST_SIDECAR_SUFFIX)) { + return false; + } + for (String prefix : dataFilePrefixes) { + if (name.startsWith(prefix) + && DATA_FILE_SUFFIX.matcher(name.substring(prefix.length())).matches()) { + return true; + } + } + return false; + } + + private static boolean isTemporary(String name) { + return name.endsWith(".tmp") || name.contains(".tmp-") || name.contains(".tmp."); + } + + // Sequential metadata can be read before it exists; a cached NotFound would hide new commits. + private static boolean isSequential(Path path) { + String name = path.getName(); + Path parent = path.getParent(); + return name.startsWith("snapshot-") + || name.startsWith("schema-") + || (name.startsWith("changelog-") + && parent != null + && "changelog".equals(parent.getName())); + } + + static List dataFilePrefixes(Options options) { + List prefixes = new ArrayList<>(); + prefixes.add(DATA_FILE_PREFIX.defaultValue()); + prefixes.add(CHANGELOG_FILE_PREFIX.defaultValue()); + for (String key : new String[] {DATA_FILE_PREFIX.key(), CHANGELOG_FILE_PREFIX.key()}) { + String prefix = options.get(key); + if (!StringUtils.isNullOrWhitespaceOnly(prefix) && !prefixes.contains(prefix)) { + prefixes.add(prefix); + } + } + return prefixes; + } + + // io-cache.targets with io-cache.target..endpoint, else io-cache.endpoint as "default" + private static Map declaredTargets(Options options) { + Map urls = new LinkedHashMap<>(); + String names = options.get("io-cache.targets"); + if (names == null) { + String url = options.get("io-cache.endpoint"); + if (url != null) { + urls.put("default", url); + } + return urls; + } + for (String name : names.toLowerCase(Locale.ROOT).split(",")) { + name = name.trim(); + if (TARGET_NAME.matcher(name).matches() && !urls.containsKey(name)) { + urls.put(name, options.get("io-cache.target." + name + ".endpoint")); + } + } + return urls; + } + + private static CacheTarget target(Options options, String name, String url) { + String prefix = "io-cache.target." + name + "."; + String region = options.get(prefix + "region"); + String pathStyle = options.get(prefix + "path-style-access"); + return new CacheTarget( + name, + url.contains("://") ? url : "https://" + url, + pathStyle != null && "true".equalsIgnoreCase(pathStyle.trim()), + StringUtils.isNullOrWhitespaceOnly(region) ? null : region); + } + + // io-cache.routes is "types=name;...": the first matching rule selects the target. + @Nullable + static String routedName(String routes, FileType type) { + for (String rule : routes.toLowerCase(Locale.ROOT).split(";")) { + int eq = rule.indexOf('='); + String name = eq > 0 ? rule.substring(eq + 1).trim() : ""; + if (!name.isEmpty() && fileTypes(rule.substring(0, eq)).contains(type)) { + return name; + } + } + return null; + } + + private static Set fileTypes(String value) { + return FileType.parseWhitelist(value.toLowerCase(Locale.ROOT)); + } + + @Override + public String toString() { + return "IoCacheRouting{" + targetByType + ", oss=" + ossEndpoint + "}"; + } + + /** The prefixes of older schemas, loaded at most once and shared by derived routings. */ + private static final class PrefixHistory implements Serializable { + + private static final long serialVersionUID = 1L; + + private final IoCacheViewableFileIO.PrefixHistory source; + @Nullable private transient volatile List loaded; + + private PrefixHistory(IoCacheViewableFileIO.PrefixHistory source) { + this.source = source; + } + + private synchronized void load(FileIO originOnly) { + if (loaded != null) { + return; + } + try { + loaded = new ArrayList<>(source.filePrefixes(originOnly)); + } catch (IOException | RuntimeException e) { + // files with a prefix missed here are read from OSS, which only loses the cache + LOG.warn("Failed to load the file prefixes of older schemas for io-cache.", e); + loaded = Collections.emptyList(); + } + } + } + + /** A named cache target with its endpoint and addressing settings. */ + static final class CacheTarget implements Serializable { + + private static final long serialVersionUID = 1L; + + final String name; + final String url; + final boolean pathStyleAccess; + @Nullable final String region; + + CacheTarget(String name, String url, boolean pathStyleAccess, @Nullable String region) { + this.name = name; + this.url = url; + this.pathStyleAccess = pathStyleAccess; + this.region = region; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CacheTarget)) { + return false; + } + CacheTarget that = (CacheTarget) o; + return name.equals(that.name) + && url.equals(that.url) + && pathStyleAccess == that.pathStyleAccess + && Objects.equals(region, that.region); + } + + @Override + public int hashCode() { + return Objects.hash(name, url, pathStyleAccess, region); + } + + @Override + public String toString() { + return name + "=" + url; + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 71468672af9f..261313159c68 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -24,6 +24,9 @@ import org.apache.paimon.fs.HadoopOptionsProvider; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; import org.apache.paimon.options.Options; import org.apache.paimon.plugin.PluginLoader; import org.apache.paimon.utils.IOUtils; @@ -40,6 +43,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -54,7 +59,8 @@ import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE; /** Jindo {@link FileIO}. */ -public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsProvider { +public class JindoFileIO extends HadoopCompliantFileIO + implements HadoopOptionsProvider, IoCacheViewableFileIO { private static final long serialVersionUID = 2L; @@ -72,6 +78,13 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; private static final String OSS_SHOW_DIR_TIMESTAMP = "fs.oss.show-dir-timestamp"; + private static final String OSS_ENDPOINT = "fs.oss.endpoint"; + private static final String OSS_HTTPS_ENABLE = "fs.oss.https.enable"; + private static final String OSS_SECOND_LEVEL_DOMAIN_ENABLE = + "fs.oss.second.level.domain.enable"; + private static final String OSS_REGION = "fs.oss.region"; + // JindoSDK options of a DLF cache cluster, which the OSS endpoint must not use + private static final String OSS_DLF_CACHE_PREFIX = "fs.oss.dlf-cache."; private static final Map CASE_SENSITIVE_KEYS = new HashMap() { @@ -91,8 +104,12 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private Options hadoopOptions; private Options hadoopOptionsWithCache; + private Map cacheTargetOptions; + transient volatile Map> cacheTargetFsMap; private boolean allowCache = true; private transient BlobPresigner blobPresigner; + // the FileIO a table view was made from; a view uses its file systems + @Nullable private JindoFileIO base; public JindoFileIO() {} @@ -163,6 +180,66 @@ public void configure(CatalogContext context) { hadoopOptionsWithCache.set("fs.oss.read.profile.columnar.use-pread", "false"); hadoopOptionsWithCache.set( "fs.jindocache.read.profile.columnar.readahead.pread.enable", "false"); + + if (cacheRouting != null) { + cacheTargetOptions = new HashMap<>(); + for (CacheTarget target : cacheRouting.targets()) { + Options options = withEndpoint(hadoopOptions, target.url); + options.set(OSS_SECOND_LEVEL_DOMAIN_ENABLE, String.valueOf(target.pathStyleAccess)); + if (target.region != null) { + options.set(OSS_REGION, target.region); + } + cacheTargetOptions.put(target.name, options); + } + // fs.oss.endpoint may name a cache for older clients, so set the OSS endpoint again + hadoopOptions = withEndpoint(hadoopOptions, cacheRouting.ossEndpoint()); + hadoopOptions.keySet().removeIf(key -> key.startsWith(OSS_DLF_CACHE_PREFIX)); + } + } + + @Override + public FileIO originOnly() { + return cacheRouting == null ? this : withRouting(cacheRouting.originOnly()); + } + + @Override + public FileIO withTableView(TableView view) { + return cacheRouting == null ? this : withRouting(cacheRouting.withTableView(view)); + } + + private JindoFileIO withRouting(IoCacheRouting routing) { + if (routing == cacheRouting) { + return this; + } + JindoFileIO view = new JindoFileIO(); + view.base = base == null ? this : base; + view.cacheRouting = routing; + view.metaCacheEnabled = metaCacheEnabled; + view.readCacheEnabled = readCacheEnabled; + view.writeCacheEnabled = writeCacheEnabled; + view.existsCacheEnabled = existsCacheEnabled; + view.allowCache = allowCache; + view.hadoopOptions = hadoopOptions; + view.hadoopOptionsWithCache = hadoopOptionsWithCache; + view.cacheTargetOptions = cacheTargetOptions; + return view; + } + + /** Copies the options with an endpoint: its host[:port], and https from its scheme if any. */ + static Options withEndpoint(Options base, @Nullable String endpoint) { + Options options = new Options(base.toMap()); + if (endpoint == null) { + return options; + } + int schemeEnd = endpoint.indexOf("://"); + if (schemeEnd >= 0) { + String scheme = endpoint.substring(0, schemeEnd); + options.set(OSS_HTTPS_ENABLE, String.valueOf("https".equalsIgnoreCase(scheme))); + endpoint = endpoint.substring(schemeEnd + 3); + } + int pathStart = endpoint.indexOf('/'); + options.set(OSS_ENDPOINT, pathStart >= 0 ? endpoint.substring(0, pathStart) : endpoint); + return options; } /** @@ -174,6 +251,12 @@ public void configure(CatalogContext context) { */ @Override public Options hadoopOptions(Path path, String opType) { + // Lance and Vortex build their own clients from these options; they only reach the origin. + return cacheRouting != null ? hadoopOptions : routedHadoopOptions(path, opType); + } + + /** The options of the endpoint that a request of this type on this path uses. */ + Options routedHadoopOptions(Path path, String opType) { boolean shouldCache = false; if (opType.equalsIgnoreCase("read")) { shouldCache = readCacheEnabled && shouldCache(path); @@ -181,29 +264,40 @@ public Options hadoopOptions(Path path, String opType) { shouldCache = writeCacheEnabled && shouldCache(path); } else if (opType.equalsIgnoreCase("meta")) { shouldCache = metaCacheEnabled && shouldCache(path); + } else if (opType.equalsIgnoreCase("exists")) { + shouldCache = existsCacheEnabled && shouldCache(path); } - if (shouldCache) { - return hadoopOptionsWithCache; - } else { + if (!shouldCache) { return hadoopOptions; + } else if (cacheRouting != null) { + return cacheTargetOptions.get(cacheTargetOf(path).name); + } else { + return hadoopOptionsWithCache; } } @Override public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite) throws IOException { - if (!overwrite && this.exists(path)) { + org.apache.hadoop.fs.Path hadoopPath = path(path); + // The check is part of the write, so a cache endpoint never sees the file before it exists. + boolean exists = + cacheRouting == null + ? this.exists(path) + : getFileSystem(hadoopPath, false).exists(hadoopPath); + if (!overwrite && exists) { throw new IOException("File " + path + " already exists."); } - org.apache.hadoop.fs.Path hadoopPath = path(path); - Pair pair = getFileSystemPair(hadoopPath, false); + boolean viaTarget = cacheRouting != null && writeCacheEnabled && shouldCache(path); + Pair pair = getFileSystemPair(hadoopPath, viaTarget); JindoHadoopSystem fs = pair.getKey(); JindoMpuStore mpuStore = fs.getMpuStore(hadoopPath); if (mpuStore == null) { LOG.debug( "Jindo multipart upload is unavailable for {}, falling back to rename commit.", path); - return super.newTwoPhaseOutputStream(path, overwrite); + // With cache routing the check above already ran on OSS; do not repeat it on a cache. + return super.newTwoPhaseOutputStream(path, overwrite || cacheRouting != null); } return new JindoTwoPhaseOutputStream( new JindoMultiPartUpload(mpuStore, fs.getWorkingDirectory()), hadoopPath, path); @@ -224,6 +318,9 @@ public String createBlobPresignedUrl( } private synchronized BlobPresigner blobPresigner() { + if (base != null) { + return base.blobPresigner(); + } if (blobPresigner == null) { PluginLoader loader = BlobPlugin.getLoader(); Thread thread = Thread.currentThread(); @@ -256,9 +353,52 @@ private static synchronized PluginLoader getLoader() { @Override protected Pair createFileSystem( org.apache.hadoop.fs.Path path, boolean enableCache) { + return createFileSystem(path, enableCache ? hadoopOptionsWithCache : hadoopOptions); + } + + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + return createFileSystem(path, cacheTargetOptions.get(target.name)); + } + + @Override + protected Pair getFileSystemPair( + org.apache.hadoop.fs.Path path, boolean enableCache) throws IOException { + CacheTarget target = + enableCache && cacheRouting != null ? cacheTargetOf(new Path(path.toUri())) : null; + if (base != null) { + return target == null + ? base.getFileSystemPair(path, false) + : base.targetFileSystemPair(path, target); + } + if (target == null) { + return super.getFileSystemPair(path, enableCache && cacheRouting == null); + } + return targetFileSystemPair(path, target); + } + + private Pair targetFileSystemPair( + org.apache.hadoop.fs.Path path, CacheTarget target) throws IOException { + if (cacheTargetFsMap == null) { + synchronized (this) { + if (cacheTargetFsMap == null) { + cacheTargetFsMap = new ConcurrentHashMap<>(); + } + } + } + String authority = path.toUri().getAuthority(); + String key = target.name + "/" + (authority == null ? "DEFAULT" : authority); + try { + return cacheTargetFsMap.computeIfAbsent(key, k -> createFileSystem(path, target)); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + + private Pair createFileSystem( + org.apache.hadoop.fs.Path path, Options options) { final String scheme = path.toUri().getScheme(); final String authority = path.toUri().getAuthority(); - Options options = enableCache ? hadoopOptionsWithCache : hadoopOptions; Supplier> supplier = () -> { Configuration hadoopConf = new Configuration(false); @@ -302,6 +442,10 @@ protected Pair createFileSystem( @Override public synchronized void close() { + if (base != null) { + // the file systems and the presigner belong to the base + return; + } if (blobPresigner != null) { Thread thread = Thread.currentThread(); ClassLoader previous = thread.getContextClassLoader(); @@ -314,8 +458,14 @@ public synchronized void close() { } } if (!allowCache) { - fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); - fsMap.clear(); + if (fsMap != null) { + fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); + fsMap.clear(); + } + if (cacheTargetFsMap != null) { + cacheTargetFsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); + cacheTargetFsMap.clear(); + } } } diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java new file mode 100644 index 000000000000..bba609f3aeda --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/IoCacheRoutingTest.java @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.FileType; +import org.apache.paimon.utils.InstantiationUtil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.paimon.utils.FileType.BUCKET_INDEX; +import static org.apache.paimon.utils.FileType.DATA; +import static org.apache.paimon.utils.FileType.FILE_INDEX; +import static org.apache.paimon.utils.FileType.GLOBAL_INDEX; +import static org.apache.paimon.utils.FileType.META; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link IoCacheRouting}. */ +public class IoCacheRoutingTest { + + private static final String TABLE_ROOT = "oss://bkt/db1.db/t1"; + private static final String UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; + private static final Path DATA_FILE = + new Path(TABLE_ROOT + "/dt=2026-09-30/bucket-0/data-" + UUID + "-0.orc"); + private static final Path MANIFEST = new Path(TABLE_ROOT + "/manifest/manifest-" + UUID + "-0"); + + // paths below are relative to TABLE_ROOT, and {uuid} stands for UUID + private static final String DATA_PATH = "dt=1/bucket-0/data-{uuid}-0.parquet"; + private static final String MANIFEST_PATH = "manifest/manifest-{uuid}-0"; + private static final String INDEX_PATH = "index/index-{uuid}-0"; + private static final String GLOBAL_INDEX_PATH = "index/btree-global-index-{uuid}.index"; + private static final String TEMP_PATH = "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp"; + private static final String SNAPSHOT_PATH = "snapshot/snapshot-12"; + private static final String LATEST_PATH = "snapshot/LATEST"; + + private static final String OSS = "https://oss-cn-hangzhou-internal.aliyuncs.com"; + private static final String CACHE = "http://cache.example.com"; + private static final String ACCEL = "https://accelerator.example.com"; + private static final String CLUSTER = "http://cluster.example.com"; + private static final String WRITE = "io-cache.policy=meta,read,write"; + private static final String EXISTS = "io-cache.policy=meta,read,exists"; + + @Test + public void testOneCacheTarget() { + Options options = single(); + assertEndpoint(options, "read", DATA_PATH, CACHE); + assertEndpoint(options, "meta", DATA_PATH, CACHE); + // a ROW sidecar is sized and read through the target of its data file + assertEndpoint(single("io-cache.whitelist=data"), "meta", DATA_PATH + ".row", CACHE); + assertEndpoint(single("io-cache.whitelist=data"), "read", DATA_PATH + ".row", CACHE); + // exists asks whether a file is still there, so it needs its own policy token + assertEndpoint(options, "exists", DATA_PATH, OSS); + assertEndpoint(single(EXISTS), "exists", DATA_PATH, CACHE); + assertEndpoint(single(EXISTS), "exists", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "read", MANIFEST_PATH, CACHE); + assertEndpoint(options, "read", "manifest/manifest-list-{uuid}-1", CACHE); + assertEndpoint(options, "read", "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE); + assertEndpoint(options, "read", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "meta", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "exists", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "read", LATEST_PATH, OSS); + assertEndpoint(options, "exists", LATEST_PATH, OSS); + assertEndpoint(options, "read", TEMP_PATH, OSS); + assertEndpoint(options, "read", "dt=1/bucket-0/000000_0", OSS); + assertEndpoint(options, "read", "dls://bkt/db1.db/t1/" + DATA_PATH, OSS); + // the whitelist has no index + assertEndpoint(options, "read", INDEX_PATH, OSS); + // a Format Table file named like a manifest may be replaced in place + assertEndpoint(options, "read", "review_external/manifest.parquet", OSS); + assertEndpoint(options, "meta", "review_external/manifest.parquet", OSS); + assertEndpoint(options, "write", DATA_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, DATA_PATH, OSS); + } + } + + @Test + public void testPolicyAndWhitelist() { + assertEndpoint(single("-io-cache.enabled"), "read", DATA_PATH, CACHE); + assertEndpoint(single("io-cache.enabled=false"), "exists", DATA_PATH, CACHE); + assertEndpoint(single("-io-cache.policy"), "exists", DATA_PATH, CACHE); + assertEndpoint(single("io-cache.policy=none"), "exists", DATA_PATH, CACHE); + + Options readOnly = single("io-cache.policy=read"); + assertEndpoint(readOnly, "read", DATA_PATH, CACHE); + assertEndpoint(readOnly, "meta", DATA_PATH, OSS); + assertEndpoint(readOnly, "exists", DATA_PATH, OSS); + Options writeOnly = single("io-cache.policy=write"); + assertEndpoint(writeOnly, "write", DATA_PATH, CACHE); + assertEndpoint(writeOnly, "exists", DATA_PATH, OSS); + Options existsOnly = single("io-cache.policy=exists"); + assertEndpoint(existsOnly, "exists", DATA_PATH, CACHE); + assertEndpoint(existsOnly, "meta", DATA_PATH, OSS); + + assertEndpoint(single("-io-cache.whitelist"), "read", INDEX_PATH, CACHE); + assertEndpoint(single("io-cache.whitelist=*"), "read", GLOBAL_INDEX_PATH, CACHE); + assertEndpoint(single("io-cache.whitelist=*"), "read", DATA_PATH + ".index", CACHE); + } + + @Test + public void testEndpoints() { + assertEndpoint(single("io-cache.endpoint= "), "read", DATA_PATH, OSS); + assertEndpoint( + single("io-cache.endpoint=", "io-cache.routes=*=default"), "read", DATA_PATH, OSS); + // the origin defaults to fs.oss.endpoint + String oss = "fs.oss.endpoint=" + OSS; + assertEndpoint(single(oss, "-io-cache.origin.endpoint"), "list", DATA_PATH, OSS); + assertEndpoint(single(oss, "io-cache.origin.endpoint= "), "list", DATA_PATH, OSS); + // a client-side endpoint turns routing off + String override = "https://oss-cn-hangzhou.aliyuncs.com"; + assertEndpoint(single("dlf.oss-endpoint=" + override), "read", DATA_PATH, override); + } + + @Test + public void testWritePolicy() { + Options options = single(WRITE); + assertEndpoint(options, "write", DATA_PATH, CACHE); + assertEndpoint(options, "write", MANIFEST_PATH, CACHE); + assertEndpoint(options, "write", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "write", LATEST_PATH, OSS); + assertEndpoint(options, "write", TEMP_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, DATA_PATH, OSS); + } + assertEndpoint(single(WRITE, "io-cache.whitelist=meta"), "write", DATA_PATH, OSS); + + assertEndpoint(multi(WRITE), "write", DATA_PATH, CLUSTER); + assertEndpoint(multi(WRITE), "write", MANIFEST_PATH, ACCEL); + } + + @Test + public void testTwoCacheTargets() { + Options options = multi(); + assertEndpoint(options, "read", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "meta", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "exists", MANIFEST_PATH, OSS); + assertEndpoint(multi(EXISTS), "exists", MANIFEST_PATH, ACCEL); + assertEndpoint(options, "read", DATA_PATH, CLUSTER); + assertEndpoint(multi(EXISTS), "exists", DATA_PATH, CLUSTER); + assertEndpoint(options, "read", INDEX_PATH, CLUSTER); + assertEndpoint(options, "read", GLOBAL_INDEX_PATH, CLUSTER); + assertEndpoint(options, "read", SNAPSHOT_PATH, OSS); + assertEndpoint(options, "write", MANIFEST_PATH, OSS); + for (String op : Arrays.asList("list", "delete", "rename", "mkdirs")) { + assertEndpoint(options, op, MANIFEST_PATH, OSS); + } + assertEndpoint(multi("io-cache.whitelist=meta"), "read", DATA_PATH, OSS); + } + + @Test + public void testTargetsAndRoutes() { + // without routes the first target with an endpoint takes every type + String noRoutes = "-io-cache.routes"; + assertEndpoint(multi(noRoutes), "read", DATA_PATH, ACCEL); + assertEndpoint(multi(noRoutes, "io-cache.endpoint=" + CACHE), "read", DATA_PATH, ACCEL); + assertEndpoint( + multi(noRoutes, "-io-cache.target.accel.endpoint"), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi(noRoutes, "io-cache.target.accel.endpoint= "), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi(noRoutes, "io-cache.targets=Bad_Name,cluster"), "read", DATA_PATH, CLUSTER); + assertEndpoint( + multi("io-cache.target.cluster.endpoint= " + CLUSTER + " "), + "read", + DATA_PATH, + CLUSTER); + + // a type without a usable route goes to the origin + assertEndpoint(multi("io-cache.routes=data=x"), "read", DATA_PATH, OSS); + assertEndpoint(multi("-io-cache.target.cluster.endpoint"), "read", DATA_PATH, OSS); + String metaAndData = "io-cache.routes=meta=accel;data=cluster"; + assertEndpoint(multi(metaAndData), "read", DATA_PATH + ".index", OSS); + + assertEndpoint(multi("io-cache.routes=*=cluster"), "read", MANIFEST_PATH, CLUSTER); + String firstWins = "io-cache.routes=data=accel;data,meta=cluster"; + assertEndpoint(multi(firstWins), "read", DATA_PATH, ACCEL); + String malformed = "io-cache.routes==cluster;meta=accel;junk"; + assertEndpoint(multi(malformed), "read", MANIFEST_PATH, ACCEL); + assertEndpoint(multi(malformed), "read", DATA_PATH, OSS); + } + + @Test + public void testDataFilesNamedLikeMetadata() { + Options manifest = multi("data-file.prefix=manifest-"); + assertEndpoint(manifest, "read", "dt=1/bucket-0/manifest-{uuid}-0.orc", CLUSTER); + assertEndpoint(manifest, "read", "dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", CLUSTER); + assertEndpoint(manifest, "read", MANIFEST_PATH, ACCEL); + assertEndpoint(manifest, "read", MANIFEST_PATH + ".avro.sidecar", ACCEL); + Options snapshot = multi("data-file.prefix=snapshot-"); + assertEndpoint(snapshot, "read", "dt=1/bucket-0/snapshot-{uuid}-0.orc", CLUSTER); + Options stat = multi("data-file.prefix=stat-"); + assertEndpoint(stat, "read", "dt=1/bucket-0/stat-{uuid}-0.orc", CLUSTER); + Options index = + multi("io-cache.routes=data=cluster;bucket-index=accel", "data-file.prefix=index-"); + assertEndpoint(index, "read", "dt=1/bucket-0/index-{uuid}-0.orc", CLUSTER); + assertEndpoint(index, "read", INDEX_PATH, ACCEL); + + String custom = "dt=1/bucket-0/custom-{uuid}-0.orc"; + assertEndpoint(single(), "read", custom, OSS); + assertEndpoint(single("data-file.prefix=custom-"), "read", custom, CACHE); + } + + @Test + public void testCacheableType() { + // named by sequence id or rewritten in place + for (String path : + Arrays.asList( + "snapshot/snapshot-12", + "snapshot/LATEST", + "snapshot/EARLIEST", + "branch/branch-dev/snapshot/LATEST", + "schema/schema-3", + "changelog/changelog-5", + "changelog/LATEST", + "tag/tag-2026-09-30", + "tag/tag-success-file/t1_SUCCESS", + "consumer/consumer-job1", + "service/service-primary-key-lookup", + "dt=1/_SUCCESS", + "oss://bkt/bucket-0/db/t/changelog/changelog-5")) { + assertType(path, null); + } + // temporary or not written by Paimon + for (String path : + Arrays.asList( + "snapshot/.snapshot-13.{uuid}.tmp", + "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp-{uuid}", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp.{uuid}", + "metadata/version-hint.text", + "metadata/v3.metadata.json", + "dt=1/bucket-0/000000_0", + "dt=1/part-00000-1a2b.snappy.parquet", + "dt=1/bucket-0/data-1.parquet", + "dt=1/bucket-0/custom-{uuid}-0.orc", + "bucket-postpone/data--u-user-s-x-w-{uuid}-0.parquet", + "bucket-postpone/data--u--s-1-w-{uuid}-0.parquet", + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.parquet", + "dt=1/bucket-0/data-{uuid}.blob", + "dt=1/bucket-0/data-{uuid}.managed.blob.{uuid}.tmp", + "dt=1/bucket-0/data-old.managed.blob", + "README")) { + assertType(path, null); + } + // Format Table files named like metadata or indexes, which may be replaced in place + for (String path : + Arrays.asList( + "dt=1/manifest.parquet", + "dt=1/stat-2024.parquet", + "dt=1/index-a.csv", + "dt=1/foo.index", + "review_external/manifest.parquet", + "manifest/manifest-old", + "manifest/manifest-old.avro.sidecar", + "manifest/manifest-list-{uuid}-1.avro.sidecar", + "manifest/manifest-list-{uuid}-changelog-1", + "manifest/manifest-{uuid}-changelog", + "statistics/stat-old", + "index/index-old", + "index/my-global-index.index", + "index/btree-global-index-{uuid}-0.index")) { + assertType(path, null); + } + + assertType("manifest/manifest-{uuid}-0", META); + assertType("manifest/manifest-list-{uuid}-1", META); + assertType("manifest/index-manifest-{uuid}-0", META); + assertType("manifest/manifest-{uuid}-0.avro.sidecar", META); + assertType("manifest/manifest-{uuid}-changelog-0", META); + assertType("manifest/manifest-{uuid}-changelog-0.avro.sidecar", META); + assertType("statistics/stat-{uuid}-0", META); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/changelog-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.blob", DATA); + assertType("dt=1/8b1f7c2e/data-{uuid}-0.parquet", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.index", FILE_INDEX); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.blobref", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.parquet.row", DATA); + assertType("dt=1/bucket-0/data-{uuid}-0.managed.blob", DATA); + assertType("dt=1/bucket-0/data-{uuid}.managed.blob", DATA); + // the postpone writer puts its commit user and write id before {uuid}-{count} + for (String postpone : + Arrays.asList( + "bucket-postpone/data--u-io-cache-user-s-7-w-{uuid}-0.parquet", + "dt=1/bucket-postpone/data--u-{uuid}-s-2147483647-w-{uuid}-12.orc", + "bucket-postpone/changelog--u-user-s-0-w-{uuid}-0.parquet")) { + assertType(postpone, DATA); + assertType(postpone + ".index", FILE_INDEX); + assertType(postpone + ".blobref", DATA); + assertType(postpone + ".row", DATA); + } + assertType("index/index-{uuid}-0", BUCKET_INDEX); + assertType("index/btree-global-index-{uuid}.index", GLOBAL_INDEX); + assertType("index/lumina-global-index-{uuid}.index", GLOBAL_INDEX); + } + + @Test + public void testCacheableTypeWithFilePrefixes() { + assertType("dt=1/bucket-0/custom-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); + assertType( + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.orc", + DATA, + "data-file.prefix=custom-"); + assertType("dt=1/bucket-0/data-{uuid}-0.orc", DATA, "data-file.prefix=custom-"); + assertType("dt=1/bucket-0/cl-{uuid}-0.orc", DATA, "changelog-file.prefix=cl-"); + assertType("dt=1/bucket-0/ unknown.parquet", null, "data-file.prefix= "); + + // the name decides, so data files may be named like metadata + String manifest = "data-file.prefix=manifest-"; + assertType("dt=1/bucket-0/manifest-{uuid}-0.orc", DATA, manifest); + assertType("dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", DATA, manifest); + assertType("dt=1/bucket-0/manifest-{uuid}-0.orc.index", FILE_INDEX, manifest); + assertType("manifest/manifest-{uuid}-0", META, manifest); + assertType("manifest/manifest-{uuid}-0.avro.sidecar", META, manifest); + String index = "data-file.prefix=index-"; + assertType("dt=1/bucket-0/index-{uuid}-0.orc", DATA, index); + assertType("bucket-postpone/index-{uuid}-0.orc", DATA, index); + assertType("index/index-{uuid}-0", BUCKET_INDEX, index); + assertType("dt=1/bucket-0/index-{uuid}-0", BUCKET_INDEX, index); + String stat = "data-file.prefix=stat-"; + assertType("dt=1/bucket-0/stat-{uuid}-0.orc", DATA, stat); + assertType("postpone/stat-{uuid}-0.orc", DATA, stat); + assertType("statistics/stat-{uuid}-0", META, stat); + String snapshot = "data-file.prefix=snapshot-"; + assertType("dt=1/bucket-0/snapshot-{uuid}-0.orc", DATA, snapshot); + assertType("dt=1/bucket-0/snapshot-{uuid}-0.orc.index", FILE_INDEX, snapshot); + assertType("dt=1/bucket-0/schema-{uuid}-0.orc", DATA, "data-file.prefix=schema-"); + assertType( + "dt=1/bucket-0/global-index-{uuid}-0.orc.index", + FILE_INDEX, + "data-file.prefix=global-index-"); + + // names rewritten in place never use a cache, whatever the prefix + assertType("dt=1/bucket-0/tag-{uuid}-0.orc", null, "data-file.prefix=tag-"); + assertType("dt=1/bucket-0/consumer-{uuid}-0.orc", null, "data-file.prefix=consumer-"); + String warehouse = "oss://bkt/warehouse/bucket-0/db/t1/"; + assertType(warehouse + "tag/tag-file", null, "data-file.prefix=tag-"); + assertType(warehouse + "consumer/consumer-file", null, "data-file.prefix=consumer-"); + assertType(warehouse + "service/service-file", null, "data-file.prefix=service-"); + assertType(warehouse + "metadata/version-hint.text", null, "data-file.prefix=version-"); + assertType(warehouse + "branch/branch-feature", null, "data-file.prefix=branch-"); + } + + @Test + public void testTableViews() { + IoCacheRouting routing = IoCacheRouting.create(multi()); + Path custom = path("dt=1/bucket-0/custom-{uuid}-0.orc"); + assertThat(routing.targetOf(custom)).isNull(); + + // a table passes the file prefixes of its schema + IoCacheRouting prefixed = routing.withTableView(view(null, "custom-", " ")); + assertThat(prefixed.targetOf(custom).name).isEqualTo("cluster"); + assertThat(prefixed.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(prefixed.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(prefixed.withTableView(view(null, "custom-"))).isSameAs(prefixed); + assertThat(routing.withTableView(TableView.NONE)).isSameAs(routing); + + // Format and Object Tables send every request to the origin + IoCacheRouting originOnly = prefixed.originOnly(); + for (Path path : Arrays.asList(custom, DATA_FILE, MANIFEST, path(INDEX_PATH))) { + assertThat(originOnly.targetOf(path)).as(path.toString()).isNull(); + } + assertThat(originOnly.originOnly()).isSameAs(originOnly); + assertThat(originOnly.withTableView(view(null, "custom-"))).isSameAs(originOnly); + assertThat(prefixed.withTableView(view(null).originOnly()).targetOf(DATA_FILE)).isNull(); + assertThat(originOnly.ossEndpoint()).isEqualTo(OSS); + assertThat(originOnly.readCacheEnabled()).isTrue(); + } + + @Test + public void testTableViewsOnlyRouteFilesUnderTheirRoots() { + IoCacheRouting routing = IoCacheRouting.create(multi()); + // what a BLOB descriptor points at may be named like a data file and still be replaced + Path imported = path("oss://bkt/import/data-{uuid}-1.png"); + Path otherTable = path("oss://bkt/db1.db/t10/dt=1/bucket-0/data-{uuid}-0.parquet"); + Path otherBucket = path("oss://other/db1.db/t1/dt=1/bucket-0/data-{uuid}-0.parquet"); + assertThat(routing.targetOf(imported).name).isEqualTo("cluster"); + + IoCacheRouting table = routing.withTableView(view(null)); + for (Path path : Arrays.asList(imported, otherTable, otherBucket)) { + assertThat(table.targetOf(path)).as(path.toString()).isNull(); + } + assertThat(table.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(table.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(table.targetOf(path(TABLE_ROOT + "/"))).isNull(); + + // an external path on the table's bucket is one of its roots + IoCacheRouting external = + routing.withTableView( + TableView.of( + Arrays.asList(new Path(TABLE_ROOT), new Path("oss://bkt/import/")), + Collections.emptyList(), + null)); + assertThat(external.targetOf(imported).name).isEqualTo("cluster"); + assertThat(external.targetOf(otherTable)).isNull(); + } + + @Test + public void testPrefixesOfOlderSchemasAreLoadedOnceWhenOnlyTheyMayRoute() { + CountingHistory history = new CountingHistory("older-"); + IoCacheRouting routing = IoCacheRouting.create(multi()).withTableView(view(history)); + Path older = path("dt=1/bucket-0/older-{uuid}-0.parquet"); + assertThat(routing.targetOf(older)).isNull(); + + // known prefixes, metadata and files outside the table never load them + for (Path path : + Arrays.asList( + DATA_FILE, + MANIFEST, + path(SNAPSHOT_PATH), + path("manifest/manifest-{uuid}-0.avro.sidecar"), + path("oss://bkt/import/older-{uuid}-0.parquet"))) { + assertThat(routing.awaitsHistory(path)).as(path.toString()).isFalse(); + } + assertThat(routing.awaitsHistory(older)).isTrue(); + routing.loadHistory(null); + routing.loadHistory(null); + assertThat(history.loads).isEqualTo(1); + assertThat(routing.awaitsHistory(older)).isFalse(); + assertThat(routing.targetOf(older).name).isEqualTo("cluster"); + // a routing derived from it shares what was loaded + assertThat(routing.withTableView(view(null, "custom-")).targetOf(older).name) + .isEqualTo("cluster"); + + // a copy on another process loads them again + IoCacheRouting copy = clone(routing); + assertThat(copy.awaitsHistory(older)).isTrue(); + + // without a target for data files nothing is loaded + Options metaOnly = multi(); + metaOnly.set("io-cache.whitelist", "meta"); + assertThat( + IoCacheRouting.create(metaOnly) + .withTableView(view(history)) + .awaitsHistory(older)) + .isFalse(); + } + + @Test + public void testFailingToLoadOlderSchemasKeepsTheirFilesOnTheOrigin() { + PrefixHistory failing = + fileIO -> { + throw new IOException("schema directory not readable"); + }; + IoCacheRouting routing = IoCacheRouting.create(multi()).withTableView(view(failing)); + Path older = path("dt=1/bucket-0/older-{uuid}-0.parquet"); + routing.loadHistory(null); + assertThat(routing.awaitsHistory(older)).isFalse(); + assertThat(routing.targetOf(older)).isNull(); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + } + + @Test + public void testTargetNames() { + Options options = singleTargetOptions(); + options.set("io-cache.targets", " Accel , cluster-1,accel,1x,a_b,,-x"); + options.set("io-cache.target.accel.endpoint", "http://a"); + options.set("io-cache.target.cluster-1.endpoint", "http://c"); + assertThat(IoCacheRouting.create(options).targets()) + .extracting(target -> target.name) + .containsExactly("accel", "cluster-1"); + + Options invalidNames = singleTargetOptions(); + invalidNames.set("io-cache.targets", "Bad_Name"); + assertThat(IoCacheRouting.create(invalidNames)).isNull(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("policyCases") + public void testCachePolicyUsesCompleteCaseInsensitiveTokens( + String policy, boolean readCache, boolean metaCache) { + Options options = singleTargetOptions(); + options.set("io-cache.policy", policy); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + assertThat(host(fileIO.routedHadoopOptions(DATA_FILE, "read").get("fs.oss.endpoint"))) + .isEqualTo(readCache ? "cache:8080" : "oss-cn-hangzhou.aliyuncs.com"); + assertThat(host(fileIO.routedHadoopOptions(DATA_FILE, "meta").get("fs.oss.endpoint"))) + .isEqualTo( + metaCache + ? "cache:8080" + : readCache + ? "origin.example.com" + : "oss-cn-hangzhou.aliyuncs.com"); + } + + private static List policyCases() { + return Arrays.asList( + Arguments.of(" READ , Meta ", true, true), + Arguments.of("read,NONE", false, false), + Arguments.of("thread,metadata", false, false), + Arguments.of("read,nonetheless", true, false)); + } + + @Test + public void testRoutedName() { + String routes = " meta = Accel ; data,BUCKET-INDEX=cluster;=x;junk;file-index=;*=all;x=y"; + assertThat(IoCacheRouting.routedName(routes, FileType.META)).isEqualTo("accel"); + assertThat(IoCacheRouting.routedName(routes, FileType.DATA)).isEqualTo("cluster"); + assertThat(IoCacheRouting.routedName(routes, FileType.BUCKET_INDEX)).isEqualTo("cluster"); + assertThat(IoCacheRouting.routedName(routes, FileType.FILE_INDEX)).isEqualTo("all"); + assertThat(IoCacheRouting.routedName("meta=accel", FileType.DATA)).isNull(); + } + + @Test + public void testTargetSettings() { + Options options = twoTargetOptions(); + options.set("io-cache.target.accel.region", "cn-hangzhou"); + options.set("io-cache.target.cluster.path-style-access", " TRUE "); + options.set("io-cache.target.off.endpoint", ""); + options.set("io-cache.targets", "accel,off,cluster,missing"); + IoCacheRouting routing = IoCacheRouting.create(options); + + assertThat(routing.targets()) + .containsExactly( + new CacheTarget("accel", "https://accel.example.com", false, "cn-hangzhou"), + new CacheTarget("cluster", "http://10.0.0.1:8080", true, null)); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + assertThat(routing.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(routing.targetOf(new Path(TABLE_ROOT + "/snapshot/snapshot-1"))).isNull(); + + Options bare = singleTargetOptions(); + bare.set("io-cache.endpoint", "cache.example.com"); + assertThat(IoCacheRouting.create(bare).targetOf(DATA_FILE).url) + .isEqualTo("https://cache.example.com"); + } + + @Test + public void testNoIoCacheRouting() { + Options override = singleTargetOptions(); + override.set("dlf.oss-endpoint", "oss-cn-hangzhou.aliyuncs.com"); + assertThat(IoCacheRouting.create(override)).isNull(); + + Options disabled = singleTargetOptions(); + disabled.set("io-cache.enabled", "false"); + assertThat(IoCacheRouting.create(disabled)).isNull(); + + Options writeOnly = singleTargetOptions(); + writeOnly.set("io-cache.policy", "write"); + IoCacheRouting writes = IoCacheRouting.create(writeOnly); + assertThat(writes.writeCacheEnabled()).isTrue(); + assertThat(writes.readCacheEnabled() || writes.metaCacheEnabled()).isFalse(); + + Options none = singleTargetOptions(); + none.remove("io-cache.endpoint"); + assertThat(IoCacheRouting.create(none)).isNull(); + + // A declared target without an endpoint sends every request to origin. + Options empty = singleTargetOptions(); + empty.set("io-cache.endpoint", ""); + IoCacheRouting routing = IoCacheRouting.create(empty); + assertThat(routing.targets()).isEmpty(); + assertThat(routing.targetOf(DATA_FILE)).isNull(); + assertThat(routing.ossEndpoint()).isEqualTo("https://origin.example.com"); + } + + @Test + public void testOssEndpoint() { + Options options = singleTargetOptions(); + assertThat(IoCacheRouting.create(options).ossEndpoint()) + .isEqualTo("https://origin.example.com"); + options.remove("io-cache.origin.endpoint"); + assertThat(IoCacheRouting.create(options).ossEndpoint()) + .isEqualTo("oss-cn-hangzhou.aliyuncs.com"); + } + + @Test + public void testSerializable() throws Exception { + IoCacheRouting routing = + InstantiationUtil.clone( + IoCacheRouting.create(twoTargetOptions()), getClass().getClassLoader()); + assertThat(routing.targetOf(MANIFEST).name).isEqualTo("accel"); + assertThat(routing.targetOf(DATA_FILE).name).isEqualTo("cluster"); + } + + private static Options singleTargetOptions() { + Options options = new Options(); + options.set("fs.oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + options.set("io-cache.enabled", "true"); + options.set("io-cache.endpoint", "http://cache:8080"); + options.set("io-cache.origin.endpoint", "https://origin.example.com"); + options.set("io-cache.policy", "meta,read"); + return options; + } + + private static Options twoTargetOptions() { + Options options = singleTargetOptions(); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", "https://accel.example.com"); + options.set("io-cache.target.cluster.endpoint", "http://10.0.0.1:8080"); + options.set("io-cache.routes", "meta=accel;data,bucket-index=cluster"); + return options; + } + + /** One cache target, which older clients also get as fs.oss.endpoint. */ + private static Options single(String... changes) { + Options options = new Options(); + options.set("fs.oss.endpoint", CACHE); + options.set("io-cache.enabled", "true"); + options.set("io-cache.endpoint", CACHE); + options.set("io-cache.origin.endpoint", OSS); + options.set("io-cache.policy", "meta,read"); + options.set("io-cache.whitelist", "meta,data"); + return with(options, changes); + } + + /** Metadata on an accelerator, data and indexes on a cache cluster. */ + private static Options multi(String... changes) { + Options options = new Options(); + options.set("fs.oss.endpoint", ACCEL); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", OSS); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", ACCEL); + options.set("io-cache.target.accel.region", "cn-hangzhou"); + options.set("io-cache.target.cluster.endpoint", CLUSTER); + options.set("io-cache.target.cluster.path-style-access", "true"); + options.set("io-cache.policy", "meta,read"); + options.set("io-cache.whitelist", "*"); + options.set( + "io-cache.routes", "meta=accel;data,bucket-index,global-index,file-index=cluster"); + return with(options, changes); + } + + // "key=value" sets an option and "-key" removes it + private static Options with(Options options, String... changes) { + for (String change : changes) { + if (change.startsWith("-")) { + options.remove(change.substring(1)); + } else { + int eq = change.indexOf('='); + options.set(change.substring(0, eq), change.substring(eq + 1)); + } + } + return options; + } + + private static void assertEndpoint(Options options, String op, String path, String expect) { + Options copy = new Options(options.toMap()); + // RESTTokenFileIO puts a client-side dlf.oss-endpoint into fs.oss.endpoint + String override = copy.get("dlf.oss-endpoint"); + if (override != null) { + copy.set("fs.oss.endpoint", override); + } + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(copy)); + assertThat(host(fileIO.routedHadoopOptions(path(path), op).get("fs.oss.endpoint"))) + .as("%s %s with %s", op, path, options.toMap()) + .isEqualTo(host(expect)); + } + + private static void assertType(String path, FileType type, String... changes) { + List prefixes = IoCacheRouting.dataFilePrefixes(with(new Options(), changes)); + assertThat(IoCacheRouting.cacheableType(path(path), prefixes)) + .as("%s with %s", path, Arrays.toString(changes)) + .isEqualTo(type); + } + + private static TableView view(@Nullable PrefixHistory history, String... prefixes) { + return TableView.of( + Collections.singletonList(new Path(TABLE_ROOT)), Arrays.asList(prefixes), history); + } + + private static IoCacheRouting clone(IoCacheRouting routing) { + try { + return InstantiationUtil.clone(routing, IoCacheRoutingTest.class.getClassLoader()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** Older schemas named their data files with these prefixes; counts the loads. */ + private static class CountingHistory implements PrefixHistory { + + private static final long serialVersionUID = 1L; + + private final List prefixes; + private int loads; + + private CountingHistory(String... prefixes) { + this.prefixes = Arrays.asList(prefixes); + } + + @Override + public List filePrefixes(FileIO fileIO) { + loads++; + return prefixes; + } + } + + private static Path path(String path) { + String resolved = path.replace("{uuid}", UUID); + return new Path(resolved.contains("://") ? resolved : TABLE_ROOT + "/" + resolved); + } + + private static String host(String endpoint) { + String host = endpoint.contains("://") ? endpoint.split("://", 2)[1] : endpoint; + return host.endsWith("/") ? host.substring(0, host.length() - 1) : host; + } +} diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java new file mode 100644 index 000000000000..b645b1fae487 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoIoCacheRoutingTest.java @@ -0,0 +1,703 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.PrefixHistory; +import org.apache.paimon.fs.iocache.IoCacheViewableFileIO.TableView; +import org.apache.paimon.jindo.IoCacheRouting.CacheTarget; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTApi; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.rest.responses.GetTableTokenResponse; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.Pair; + +import com.aliyun.jindodata.api.spec.JdoException; +import com.aliyun.jindodata.common.JindoHadoopSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for the cache endpoints of {@link JindoFileIO}. */ +public class JindoIoCacheRoutingTest { + + private static final String TABLE = "oss://bkt/db1.db/t1"; + private static final String UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; + private static final Path DATA = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-1.orc"); + private static final Path MANIFEST = new Path(TABLE + "/manifest/manifest-" + UUID + "-1"); + private static final Path SNAPSHOT = new Path(TABLE + "/snapshot/snapshot-2"); + private static final Path LATEST = new Path(TABLE + "/snapshot/LATEST"); + private static final String ACCEL_HOST = "cn-hangzhou-j-internal.oss-data-acc.aliyuncs.com"; + private static final String CLUSTER_HOST = "10.0.0.1:8080"; + private static final String OSS_HOST = "oss-cn-hangzhou-internal.aliyuncs.com"; + private static final String DLF_CACHE_KEY = "fs.oss.dlf-cache.consistent-hash.enabled"; + + private JindoHadoopSystem accelFs; + private JindoHadoopSystem clusterFs; + private JindoHadoopSystem ossFs; + + @BeforeEach + public void before() throws IOException { + accelFs = mockFileSystem(); + clusterFs = mockFileSystem(); + ossFs = mockFileSystem(); + } + + @Test + public void testOptionsOfEachCacheTarget() { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + Options cluster = fileIO.routedHadoopOptions(DATA, "read"); + assertThat(cluster.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); + assertThat(cluster.get("fs.oss.https.enable")).isEqualTo("false"); + assertThat(cluster.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); + assertThat(cluster.get("fs.oss.region")).isEqualTo("cn-hangzhou"); + assertThat(cluster.get(DLF_CACHE_KEY)).isEqualTo("true"); + assertThat(cluster.containsKey("fs.xengine")).isFalse(); + assertThat(fileIO.routedHadoopOptions(DATA, "meta")).isSameAs(cluster); + + Options accel = fileIO.routedHadoopOptions(MANIFEST, "meta"); + assertThat(accel.get("fs.oss.endpoint")).isEqualTo(ACCEL_HOST); + assertThat(accel.get("fs.oss.https.enable")).isEqualTo("true"); + assertThat(accel.get("fs.oss.second.level.domain.enable")).isEqualTo("false"); + assertThat(accel.get("fs.oss.region")).isEqualTo("cn-shanghai"); + + Options oss = fileIO.routedHadoopOptions(DATA, "write"); + assertThat(oss.get("fs.oss.endpoint")).isEqualTo(OSS_HOST); + assertThat(oss.get("fs.oss.https.enable")).isEqualTo("true"); + assertThat(oss.get("fs.oss.region")).isEqualTo("cn-hangzhou"); + assertThat(oss.containsKey(DLF_CACHE_KEY)).isFalse(); + for (Path path : + new Path[] {SNAPSHOT, LATEST, new Path(TABLE + "/dt=1/bucket-0/000000_0")}) { + assertThat(fileIO.routedHadoopOptions(path, "read")).as(path.toString()).isSameAs(oss); + } + assertThat(fileIO.routedHadoopOptions(DATA, "list")).isSameAs(oss); + } + + @Test + public void testLanceAndVortexGetTheOssOptions() { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + // their converters build an https virtual-hosted address, which only the origin serves + Options oss = fileIO.routedHadoopOptions(DATA, "write"); + for (String op : new String[] {"read", "write", "meta"}) { + for (Path path : new Path[] {DATA, MANIFEST, LATEST}) { + assertThat(fileIO.hadoopOptions(path, op)).as(path + " " + op).isSameAs(oss); + } + } + } + + @Test + public void testOnlyReadsAndStatusUseIoCacheRouting() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path copy = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-2.orc"); + + fileIO.newInputStream(DATA).close(); + fileIO.getFileSize(DATA); + fileIO.getFileStatus(MANIFEST); + fileIO.exists(DATA); + fileIO.exists(MANIFEST); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(clusterFs).getFileStatus(any()); + verify(clusterFs).exists(any()); + verify(accelFs).getFileStatus(any()); + verify(accelFs).exists(any()); + + fileIO.exists(SNAPSHOT); + fileIO.newInputStream(SNAPSHOT).close(); + fileIO.getFileStatus(SNAPSHOT); + fileIO.newOutputStream(DATA, false); + fileIO.listStatus(DATA.getParent()); + fileIO.delete(DATA, false); + fileIO.mkdirs(DATA.getParent()); + fileIO.rename(DATA, copy); + verify(ossFs).exists(any()); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + verify(ossFs).create(any(), anyBoolean()); + verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).delete(any(), anyBoolean()); + verify(ossFs).mkdirs(any()); + verify(ossFs).rename(any(), any()); + + // the two-phase write checks a routable data file on OSS, not on its cache target + fileIO.tryToWriteAtomic(new Path(TABLE + "/snapshot/snapshot-3"), "{}"); + fileIO.newTwoPhaseOutputStream(copy, false); + verify(ossFs, times(3)).create(any(), anyBoolean()); + verify(ossFs, times(2)).exists(any()); + verify(ossFs, times(2)).rename(any(), any()); + verify(ossFs).getMpuStore(any()); + + verify(clusterFs).exists(any()); + for (JindoHadoopSystem fs : new JindoHadoopSystem[] {accelFs, clusterFs}) { + verify(fs, never()).create(any(), anyBoolean()); + verify(fs, never()).listStatus(any(org.apache.hadoop.fs.Path.class)); + verify(fs, never()).rename(any(), any()); + } + } + + @Test + public void testExistsUsesOssWithoutTheExistsToken() throws IOException { + Options options = cacheTargetOptions(); + options.set("io-cache.policy", "meta,read"); + JindoFileIO fileIO = configuredFileIO(options); + + fileIO.exists(DATA); + fileIO.exists(MANIFEST); + fileIO.getFileStatus(DATA); + verify(ossFs, times(2)).exists(any()); + verify(clusterFs).getFileStatus(any()); + verify(clusterFs, never()).exists(any()); + verify(accelFs, never()).exists(any()); + } + + @Test + public void testCacheTargetErrorsAreThrown() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + IOException unavailable = jindoError(6503, "open failed: 503 Service Unavailable"); + FileNotFoundException notFound = new FileNotFoundException("404"); + when(clusterFs.open(any(org.apache.hadoop.fs.Path.class))).thenThrow(unavailable); + when(accelFs.getFileStatus(any())).thenThrow(notFound); + + assertThatThrownBy(() -> fileIO.newInputStream(DATA)).isSameAs(unavailable); + assertThatThrownBy(() -> fileIO.getFileStatus(MANIFEST)).isSameAs(notFound); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).getFileStatus(any()); + } + + @Test + public void testCacheTargetInitializationErrorsStayChecked() throws IOException { + SocketTimeoutException timeout = new SocketTimeoutException("connect timed out"); + JindoFileIO fileIO = + new TestingJindoFileIO(accelFs, clusterFs, ossFs) { + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + // JindoFileIO wraps an initialization IOException the same way + throw new UncheckedIOException(timeout); + } + }; + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + assertThatThrownBy(() -> fileIO.newInputStream(DATA)).isSameAs(timeout); + assertThatThrownBy(() -> fileIO.getFileStatus(MANIFEST)).isSameAs(timeout); + assertThatThrownBy(() -> fileIO.exists(MANIFEST)).isSameAs(timeout); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testWritesFollowTheRouteWithWritePolicy() throws IOException { + Options options = cacheTargetOptions(); + options.set("io-cache.policy", "meta,read,write"); + JindoFileIO fileIO = configuredFileIO(options); + Path next = new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + "-2.orc"); + + fileIO.newOutputStream(DATA, false); + fileIO.newOutputStream(MANIFEST, false); + fileIO.newOutputStream(SNAPSHOT, false); + fileIO.newTwoPhaseOutputStream(next, false); + assertThat(fileIO.routedHadoopOptions(DATA, "write").get("fs.oss.endpoint")) + .isEqualTo(CLUSTER_HOST); + verify(clusterFs).create(any(), anyBoolean()); + verify(accelFs).create(any(), anyBoolean()); + verify(clusterFs).getMpuStore(any()); + // the snapshot, the existence check before the two-phase write and the temp file of its + // rename fallback (no multipart upload here) stay on OSS + verify(ossFs, times(2)).create(any(), anyBoolean()); + verify(ossFs).exists(any()); + verify(clusterFs, never()).exists(any()); + + fileIO.delete(DATA, false); + fileIO.rename(DATA, next); + fileIO.listStatus(DATA.getParent()); + verify(ossFs).delete(any(), anyBoolean()); + verify(ossFs).rename(any(), any()); + verify(ossFs).listStatus(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testTokenFileIOsOfOneTokenKeepTheirCatalogEndpoints() throws IOException { + Identifier identifier = Identifier.create("db1", "t1"); + Map token = new HashMap<>(); + token.put("fs.oss.accessKeyId", "ak-" + java.util.UUID.randomUUID()); + token.put("fs.oss.accessKeySecret", "sk"); + // the cache older clients use + token.put("fs.oss.endpoint", ACCEL_HOST); + GetTableTokenResponse response = new GetTableTokenResponse(token, Long.MAX_VALUE); + String otherOss = "oss-cn-shanghai-internal.aliyuncs.com"; + String otherCluster = "10.0.0.2:8080"; + + JindoFileIO first = jindoOf(tokenFileIO(identifier, response, OSS_HOST, CLUSTER_HOST)); + JindoFileIO second = jindoOf(tokenFileIO(identifier, response, otherOss, otherCluster)); + + assertThat(endpoint(first.routedHadoopOptions(DATA, "read"))).isEqualTo(CLUSTER_HOST); + assertThat(endpoint(first.routedHadoopOptions(DATA, "write"))).isEqualTo(OSS_HOST); + assertThat(endpoint(second.routedHadoopOptions(DATA, "read"))).isEqualTo(otherCluster); + assertThat(endpoint(second.routedHadoopOptions(DATA, "write"))).isEqualTo(otherOss); + assertThat(endpoint(second.hadoopOptions(DATA, "read"))).isEqualTo(otherOss); + assertThat(jindoOf(tokenFileIO(identifier, response, OSS_HOST, CLUSTER_HOST))) + .isSameAs(first); + } + + private static RESTTokenFileIO tokenFileIO( + Identifier identifier, GetTableTokenResponse response, String origin, String target) { + RESTApi api = mock(RESTApi.class); + when(api.loadTableToken(identifier)).thenReturn(response); + Options options = new Options(); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", origin); + options.set("io-cache.endpoint", "http://" + target); + options.set("io-cache.target.default.path-style-access", "true"); + options.set("io-cache.policy", "meta,read"); + return new RESTTokenFileIO( + CatalogContext.create(options), api, identifier, new Path(TABLE)); + } + + private static JindoFileIO jindoOf(RESTTokenFileIO fileIO) throws IOException { + return (JindoFileIO) fileIO.fileIO(); + } + + private static String endpoint(Options options) { + return options.get("fs.oss.endpoint"); + } + + @Test + public void testOriginOnlyViewSendsEveryRequestToOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + FileIO view = fileIO.originOnly(); + assertThat(view).isInstanceOf(JindoFileIO.class).isNotSameAs(fileIO); + assertThat(IoCacheViewableFileIO.originOnly(view)).isSameAs(view); + assertThat(IoCacheViewableFileIO.withTableView(view, view("custom-"))).isSameAs(view); + + view.newInputStream(DATA).close(); + view.getFileStatus(MANIFEST); + view.exists(DATA); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + verify(ossFs).exists(any()); + Options oss = fileIO.routedHadoopOptions(DATA, "write"); + assertThat(((JindoFileIO) view).routedHadoopOptions(DATA, "read")).isSameAs(oss); + assertThat(((JindoFileIO) view).routedHadoopOptions(MANIFEST, "meta")).isSameAs(oss); + + // the base keeps its routes + fileIO.newInputStream(DATA).close(); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(accelFs, never()).getFileStatus(any()); + } + + @Test + public void testFilePrefixViewRoutesDataFilesNamedWithThem() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path custom = new Path(TABLE + "/dt=1/bucket-0/custom-" + UUID + "-1.orc"); + assertThat(fileIO.withTableView(TableView.NONE)).isSameAs(fileIO); + + fileIO.newInputStream(custom).close(); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + + FileIO view = fileIO.withTableView(view("custom-")); + assertThat(IoCacheViewableFileIO.withTableView(view, view("custom-"))).isSameAs(view); + view.newInputStream(custom).close(); + view.newInputStream(DATA).close(); + view.getFileStatus(MANIFEST); + verify(clusterFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(accelFs).getFileStatus(any()); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testPostponeFilesAndBlobPacksUseTheirTarget() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + String postpone = "/bucket-postpone/data--u-user-s-7-w-" + UUID + "-0.avro"; + fileIO.newInputStream(new Path(TABLE + postpone)).close(); + fileIO.newInputStream(new Path(TABLE + "/dt=1/bucket-0/data-" + UUID + ".managed.blob")) + .close(); + verify(clusterFs, times(2)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testTableViewSendsWhatItsRowsPointAtOutsideTheTableToOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + // a BLOB descriptor in a row of the table, named like a data file + Path imported = new Path("oss://bkt/import/data-" + UUID + "-1.png"); + FileIO view = fileIO.withTableView(view()); + + view.newInputStream(imported).close(); + view.getFileStatus(imported); + verify(ossFs).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs).getFileStatus(any()); + view.newInputStream(DATA).close(); + verify(clusterFs).open(any(org.apache.hadoop.fs.Path.class)); + assertThat(((JindoFileIO) view).routedHadoopOptions(imported, "read")) + .isSameAs(fileIO.routedHadoopOptions(DATA, "write")); + } + + @Test + public void testPrefixesOfOlderSchemasAreReadFromOss() throws IOException { + JindoFileIO fileIO = configuredFileIO(cacheTargetOptions()); + Path older = new Path(TABLE + "/dt=1/bucket-0/older-" + UUID + "-1.orc"); + List readers = new ArrayList<>(); + PrefixHistory history = + reader -> { + readers.add(reader); + reader.exists(new Path(TABLE + "/schema")); + return Collections.singletonList("older-"); + }; + FileIO view = + fileIO.withTableView( + TableView.of( + Collections.singletonList(new Path(TABLE)), + Collections.emptyList(), + history)); + + view.newInputStream(DATA).close(); + assertThat(readers).isEmpty(); + view.newInputStream(older).close(); + view.newInputStream(older).close(); + assertThat(readers).hasSize(1); + // the schemas are listed on OSS, the data file is read from its target + verify(ossFs).exists(any()); + verify(clusterFs, times(3)).open(any(org.apache.hadoop.fs.Path.class)); + verify(ossFs, never()).open(any(org.apache.hadoop.fs.Path.class)); + } + + @Test + public void testViewsUseTheFileSystemsOfTheirBase() throws Exception { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = configuredFileIO(options); + FileIO view = IoCacheViewableFileIO.originOnly(fileIO.withTableView(view("custom-"))); + + view.newInputStream(DATA).close(); + assertThat(fileIO.fsMap).hasSize(1); + view.close(); + verify(ossFs, never()).close(); + fileIO.close(); + verify(ossFs).close(); + + // a serialized view keeps its routes + JindoFileIO plain = new JindoFileIO(); + plain.configure(CatalogContext.create(cacheTargetOptions())); + Path custom = new Path(TABLE + "/dt=1/bucket-0/custom-" + UUID + "-1.orc"); + JindoFileIO copy = + InstantiationUtil.clone( + (JindoFileIO) plain.withTableView(view("custom-")), + getClass().getClassLoader()); + assertThat(copy.routedHadoopOptions(custom, "read").get("fs.oss.endpoint")) + .isEqualTo(CLUSTER_HOST); + assertThat(plain.routedHadoopOptions(custom, "read").get("fs.oss.endpoint")) + .isEqualTo(OSS_HOST); + } + + @Test + public void testSingleCacheTarget() { + Options options = baseOptions(); + options.set("io-cache.endpoint", "http://" + CLUSTER_HOST); + options.set("io-cache.target.default.path-style-access", "true"); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + + Options read = fileIO.routedHadoopOptions(DATA, "read"); + assertThat(read.get("fs.oss.endpoint")).isEqualTo(CLUSTER_HOST); + assertThat(read.get("fs.oss.second.level.domain.enable")).isEqualTo("true"); + assertThat(fileIO.routedHadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.routedHadoopOptions(DATA, "write").get("fs.oss.endpoint")) + .isEqualTo(OSS_HOST); + } + + @Test + public void testJindoCacheTakesPrecedence() { + Options options = cacheTargetOptions(); + options.set("fs.jindocache.namespace.rpc.address", "rpc:8101"); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + + assertThat(fileIO.cacheRouting).isNull(); + Options read = fileIO.routedHadoopOptions(DATA, "read"); + assertThat(read.get("fs.xengine")).isEqualTo("jindocache"); + assertThat(read.get("fs.oss.endpoint")).isEqualTo("http://" + CLUSTER_HOST); + Options write = fileIO.routedHadoopOptions(DATA, "write"); + assertThat(write.containsKey("fs.xengine")).isFalse(); + // legacy whitelist-path matching: "manifest" is in the default whitelist + assertThat(fileIO.routedHadoopOptions(MANIFEST, "meta")).isSameAs(read); + assertThat(fileIO.routedHadoopOptions(LATEST, "read")).isSameAs(write); + } + + @Test + public void testWithoutIoCacheRouting() { + Options override = cacheTargetOptions(); + override.set("dlf.oss-endpoint", "oss-cn-hangzhou.aliyuncs.com"); + override.set("fs.oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + Options notEnabled = cacheTargetOptions(); + notEnabled.remove("io-cache.enabled"); + Options noPolicy = cacheTargetOptions(); + noPolicy.set("io-cache.policy", "prefetch"); + + for (Options options : new Options[] {override, notEnabled, noPolicy}) { + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + assertThat(fileIO.cacheRouting).isNull(); + assertThat(fileIO.originOnly()).isSameAs(fileIO); + assertThat(fileIO.withTableView(view("custom-"))).isSameAs(fileIO); + for (String op : new String[] {"read", "meta", "write"}) { + Options hadoopOptions = fileIO.routedHadoopOptions(DATA, op); + assertThat(hadoopOptions.get("fs.oss.endpoint")) + .isEqualTo(options.get("fs.oss.endpoint")); + assertThat(hadoopOptions.containsKey("fs.oss.https.enable")).isFalse(); + } + } + } + + @Test + public void testWithEndpoint() { + Options base = new Options(); + base.set("fs.oss.https.enable", "false"); + + Options bare = JindoFileIO.withEndpoint(base, "oss.example.com"); + assertThat(bare.get("fs.oss.endpoint")).isEqualTo("oss.example.com"); + assertThat(bare.get("fs.oss.https.enable")).isEqualTo("false"); + + Options https = JindoFileIO.withEndpoint(base, "HTTPS://cache.example.com:443/"); + assertThat(https.get("fs.oss.endpoint")).isEqualTo("cache.example.com:443"); + assertThat(https.get("fs.oss.https.enable")).isEqualTo("true"); + + Options http = JindoFileIO.withEndpoint(base, "http://10.0.0.1:8080"); + assertThat(http.get("fs.oss.endpoint")).isEqualTo("10.0.0.1:8080"); + assertThat(http.get("fs.oss.https.enable")).isEqualTo("false"); + + assertThat(JindoFileIO.withEndpoint(base, null).toMap()).isEqualTo(base.toMap()); + } + + @Test + public void testFileSystemPerCacheTargetAndBucket() throws IOException { + List created = new ArrayList<>(); + JindoFileIO fileIO = + new JindoFileIO() { + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + created.add(target.name + "/" + path.toUri().getAuthority()); + return Pair.of(mock(JindoHadoopSystem.class), "oss"); + } + }; + fileIO.configure(CatalogContext.create(cacheTargetOptions())); + + Pair first = + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/manifest/manifest-" + UUID + "-1"), true); + assertThat( + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/manifest/manifest-" + UUID + "-2"), true)) + .isSameAs(first); + fileIO.getFileSystemPair( + hadoopPath("oss://other/t/manifest/manifest-" + UUID + "-3"), true); + fileIO.getFileSystemPair( + hadoopPath(TABLE + "/dt=1/bucket-0/data-" + UUID + "-3.orc"), true); + assertThat(created).containsExactly("accel/bkt", "accel/other", "cluster/bkt"); + } + + @Test + public void testCloseAfterOnlyCacheReads() throws IOException { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = new TestingJindoFileIO(accelFs, clusterFs, ossFs); + fileIO.configure(CatalogContext.create(options)); + fileIO.newInputStream(DATA).close(); + + fileIO.close(); + + verify(clusterFs).close(); + } + + @Test + public void testCloseClosesCacheTargetFileSystems() throws IOException { + Options options = cacheTargetOptions(); + options.set(FILE_IO_ALLOW_CACHE, false); + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(CatalogContext.create(options)); + fileIO.fsMap = new ConcurrentHashMap<>(); + fileIO.fsMap.put("bkt", Pair.of(ossFs, "oss")); + fileIO.cacheTargetFsMap = new ConcurrentHashMap<>(); + fileIO.cacheTargetFsMap.put("accel/bkt", Pair.of(accelFs, "oss")); + fileIO.cacheTargetFsMap.put("cluster/bkt", Pair.of(clusterFs, "oss")); + + fileIO.close(); + + for (JindoHadoopSystem fs : new JindoHadoopSystem[] {ossFs, accelFs, clusterFs}) { + verify(fs).close(); + } + assertThat(fileIO.cacheTargetFsMap).isEmpty(); + } + + private static Options baseOptions() { + Options options = new Options(); + options.set("fs.oss.endpoint", "http://" + CLUSTER_HOST); + options.set("fs.oss.accessKeyId", "ak"); + options.set("fs.oss.accessKeySecret", "sk"); + options.set("fs.oss.region", "cn-hangzhou"); + options.set("io-cache.enabled", "true"); + options.set("io-cache.origin.endpoint", "https://" + OSS_HOST); + options.set("io-cache.policy", "meta,read"); + options.set(DLF_CACHE_KEY, "true"); + return options; + } + + private static Options cacheTargetOptions() { + Options options = baseOptions(); + options.set("io-cache.targets", "accel,cluster"); + options.set("io-cache.target.accel.endpoint", "https://" + ACCEL_HOST); + options.set("io-cache.target.accel.region", "cn-shanghai"); + options.set("io-cache.target.cluster.endpoint", "http://" + CLUSTER_HOST); + options.set("io-cache.target.cluster.path-style-access", "true"); + options.set("io-cache.routes", "meta=accel;data,bucket-index=cluster"); + options.set("io-cache.policy", "meta,read,exists"); + return options; + } + + private static TableView view(String... prefixes) { + return TableView.of( + Collections.singletonList(new Path(TABLE)), Arrays.asList(prefixes), null); + } + + private JindoFileIO configuredFileIO(Options options) { + JindoFileIO fileIO = new TestingJindoFileIO(accelFs, clusterFs, ossFs); + fileIO.configure(CatalogContext.create(options)); + return fileIO; + } + + private static IOException jindoError(int code, String message) { + return new IOException(new JdoException(code, message)); + } + + // Hadoop's Path(String) needs commons-lang, which is not on this module's test classpath. + private static org.apache.hadoop.fs.Path hadoopPath(String path) { + return new org.apache.hadoop.fs.Path(URI.create(path)); + } + + private static JindoHadoopSystem mockFileSystem() throws IOException { + JindoHadoopSystem fs = mock(JindoHadoopSystem.class); + when(fs.open(any(org.apache.hadoop.fs.Path.class))) + .thenAnswer(invocation -> new FSDataInputStream(new BytesInput())); + when(fs.create(any(), anyBoolean())).thenReturn(mock(FSDataOutputStream.class)); + when(fs.getFileStatus(any())).thenReturn(mock(FileStatus.class)); + when(fs.rename(any(), any())).thenReturn(true); + return fs; + } + + private static class BytesInput extends ByteArrayInputStream + implements Seekable, PositionedReadable { + + private BytesInput() { + super(new byte[] {1, 2, 3}); + } + + @Override + public void seek(long position) { + pos = (int) position; + } + + @Override + public long getPos() { + return pos; + } + + @Override + public boolean seekToNewSource(long targetPos) { + return false; + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void readFully(long position, byte[] buffer) { + throw new UnsupportedOperationException(); + } + } + + private static class TestingJindoFileIO extends JindoFileIO { + + private final JindoHadoopSystem accelFs; + private final JindoHadoopSystem clusterFs; + private final JindoHadoopSystem ossFs; + + private TestingJindoFileIO( + JindoHadoopSystem accelFs, JindoHadoopSystem clusterFs, JindoHadoopSystem ossFs) { + this.accelFs = accelFs; + this.clusterFs = clusterFs; + this.ossFs = ossFs; + } + + @Override + protected Pair createFileSystem( + org.apache.hadoop.fs.Path path, boolean enableCache) { + assertThat(enableCache).isFalse(); + return Pair.of(ossFs, "oss"); + } + + @Override + Pair createFileSystem( + org.apache.hadoop.fs.Path path, CacheTarget target) { + return Pair.of("accel".equals(target.name) ? accelFs : clusterFs, "oss"); + } + } +} diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java index 19a3da7bd533..c09b9ce71153 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/TestJindoCacheEnable.java @@ -147,6 +147,22 @@ public void testCacheDisabledWhenPolicyNull() { verifyCacheFlags(fileIO, false, false, false); } + @Test + public void testLegacyPolicyMatchingRemainsUnchanged() { + // JindoCache RPC policies intentionally keep their historical substring matching. + JindoFileIO fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, "read,nonetheless", true, null)); + verifyCacheFlags(fileIO, false, false, false); + + fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, "thread,metadata", true, null)); + verifyCacheFlags(fileIO, true, true, false); + + fileIO = new JindoFileIO(); + fileIO.configure(createCatalogContext(true, " READ , Meta ", true, null)); + verifyCacheFlags(fileIO, false, false, false); + } + @Test public void testCacheWhitelist() { // default config