From 7fe0c09155ec8bc3c8408617c23e5bf024acd646 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 23 Jul 2026 18:32:51 +0800 Subject: [PATCH 1/5] [core][flink][spark] Support OSS Blob presigned URLs --- docs/docs/multimodal-table/blob.mdx | 75 ++++ docs/docs/spark/sql-functions.md | 34 ++ .../java/org/apache/paimon/data/Blob.java | 7 + .../java/org/apache/paimon/fs/FileIO.java | 8 + .../org/apache/paimon/fs/PluginFileIO.java | 13 + .../org/apache/paimon/fs/ResolvingFileIO.java | 13 + .../apache/paimon/fs/cache/CachingFileIO.java | 9 + .../apache/paimon/rest/RESTTokenFileIO.java | 12 + .../paimon/utils/BlobDescriptorUtils.java | 32 ++ .../java/org/apache/paimon/data/BlobTest.java | 40 ++ .../apache/paimon/fs/PluginFileIOTest.java | 86 ++++ .../apache/paimon/fs/ResolvingFileIOTest.java | 28 ++ .../paimon/fs/cache/CachingFileIOTest.java | 26 ++ .../paimon/rest/RESTTokenFileIOTest.java | 84 ++++ .../paimon/utils/BlobDescriptorUtilsTest.java | 59 +++ paimon-filesystems/paimon-oss-impl/pom.xml | 7 + .../java/org/apache/paimon/oss/OSSFileIO.java | 229 ++++++++++ .../org/apache/paimon/oss/OSSFileIOTest.java | 391 ++++++++++++++++++ .../flink/function/BuiltInFunctions.java | 6 + .../DescriptorToPresignedUrlFunction.java | 186 +++++++++ .../TryDescriptorToPresignedUrlFunction.java | 43 ++ .../apache/paimon/flink/BlobTableITCase.java | 32 ++ .../DescriptorToPresignedUrlFunctionTest.java | 190 +++++++++ .../org/apache/paimon/spark/SparkCatalog.java | 2 +- .../DescriptorToPresignedUrlFunction.java | 96 +++++ .../DescriptorToPresignedUrlUnbound.java | 99 +++++ ...olvedDescriptorToPresignedUrlFunction.java | 124 ++++++ .../catalog/functions/PaimonFunctions.scala | 17 +- .../analysis/PaimonFunctionResolver.scala | 77 +++- .../analysis/ReplacePaimonFunctions.scala | 123 +++++- .../DescriptorToPresignedUrlFunctionTest.java | 201 +++++++++ .../paimon/spark/sql/PaimonFunctionTest.scala | 59 +++ 32 files changed, 2402 insertions(+), 6 deletions(-) create mode 100644 paimon-common/src/test/java/org/apache/paimon/fs/PluginFileIOTest.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/utils/BlobDescriptorUtilsTest.java create mode 100644 paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunction.java create mode 100644 paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/TryDescriptorToPresignedUrlFunction.java create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunctionTest.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunction.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlUnbound.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/ResolvedDescriptorToPresignedUrlFunction.java create mode 100644 paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunctionTest.java diff --git a/docs/docs/multimodal-table/blob.mdx b/docs/docs/multimodal-table/blob.mdx index 62aa097f8956..3a8e4c520c87 100644 --- a/docs/docs/multimodal-table/blob.mdx +++ b/docs/docs/multimodal-table/blob.mdx @@ -502,6 +502,81 @@ CREATE TABLE descriptor_table ( Paimon stores only serialized `BlobDescriptor` bytes in normal data files. Reading the blob follows the descriptor URI to access bytes, and writing requires descriptor input for those fields. +## Presigned URLs for OSS Blobs + +Paimon can create a temporary HTTPS URL for an OSS-backed `BlobDescriptor`. The descriptor may +refer to a byte range inside a larger `.blob` file. Paimon first materializes exactly that range as +a separate OSS object, sets its content type from the requested extension, and then returns a +presigned GET URL. + +Configure the catalog with the standard public HTTPS OSS endpoint. Internal endpoints and +endpoints without the `https` scheme are rejected because external consumers must be able to fetch +the returned URL: + +```properties +fs.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com +``` + +To obtain descriptors from a regular BLOB column, read the source table with +`blob-as-descriptor=true`. The SQL functions have strict and error-tolerant forms: + +```sql +-- Flink SQL +SELECT sys.descriptor_to_presigned_url( + 'default.image_table', + image, + 'png', + INTERVAL '5' MINUTE) +FROM image_table /*+ OPTIONS('blob-as-descriptor'='true') */; + +SELECT sys.try_descriptor_to_presigned_url( + 'default.image_table', + image, + 'png', + INTERVAL '5' MINUTE) +FROM image_table /*+ OPTIONS('blob-as-descriptor'='true') */; +``` + +```sql +-- Spark SQL +ALTER TABLE image_table SET TBLPROPERTIES ('blob-as-descriptor' = 'true'); + +SELECT sys.descriptor_to_presigned_url( + 'default.image_table', + image, + 'png', + INTERVAL '5' MINUTE) +FROM image_table; +``` + +In both Flink and Spark, `source_table` must be a non-null string literal in +`database.table` form. `catalog.database.table` is also accepted when the catalog matches the +catalog that owns the function. Dynamic columns and `CASE` expressions are rejected during +planning. The strict function propagates row errors; `try_descriptor_to_presigned_url` returns +`NULL` for row-level errors. The validity interval must be positive whole seconds. + +The Java API uses the same table-root safety check: + +```java +DataTable table = ...; +Blob blob = ...; +String url = + blob.toPresignedUrl( + table.fileIO(), table.location(), "png", Duration.ofMinutes(5)); +``` + +The extension must contain only lower-case letters or digits after normalization and must map to a +known MIME type. It must match the actual content. For direct model `image_url` inputs, use only +image formats verified with the target model; PDF is not covered by this entry point. + +Materialized objects are keyed by the complete descriptor fingerprint and extension. Repeated +short-term calls for the same descriptor reuse an object whose length, content type, and +fingerprint still match, while generating a fresh URL. The first version does not resolve races +with stale-cache cleanup or orphan-file cleaning. + +Treat the returned URL as a bearer credential: submit it immediately to the external service, do +not persist it, and never write the URL or its signature/security-token query parameters to logs. + ## Blob View Blob view is useful when a downstream table should reference BLOB values already stored in an upstream table, without copying the bytes or creating new `.blob` files. A blob view field stores only a small `BlobViewStruct` inline. When the field is read, Paimon resolves the referenced BLOB from the upstream table. diff --git a/docs/docs/spark/sql-functions.md b/docs/docs/spark/sql-functions.md index 7e94c46a3ffc..56bd8862fa02 100644 --- a/docs/docs/spark/sql-functions.md +++ b/docs/docs/spark/sql-functions.md @@ -94,6 +94,40 @@ SELECT sys.descriptor_to_string(content) FROM t WHERE id = '1'; -- [BlobDescriptor{version=1', uri='/path/to/data-2c103f6f-3857-4062-abc3-2e260374a68e-1.blob', offset=4, length=1048576}] ``` +### descriptor_to_presigned_url + +`sys.descriptor_to_presigned_url($source_table, $descriptor, $extension, $validity)` + +Creates a temporary HTTPS GET URL for an OSS-backed blob descriptor. Use +`sys.try_descriptor_to_presigned_url` to return `NULL` instead of failing on row-level errors. + +- `source_table` must be a non-null STRING literal in `database.table` form, or + `catalog.database.table` with the same catalog as the function. +- `descriptor` is the serialized BINARY descriptor. Set `blob-as-descriptor=true` when reading a + normal BLOB column. +- `extension` determines the materialized object's MIME type and must match the content. +- `validity` is a positive day-time interval containing whole seconds. + +The catalog's `fs.oss.endpoint` must be a standard public HTTPS endpoint, for example +`https://oss-cn-hangzhou.aliyuncs.com`. + +```sql +ALTER TABLE image_table SET TBLPROPERTIES ('blob-as-descriptor' = 'true'); + +SELECT sys.descriptor_to_presigned_url( + 'default.image_table', + image, + 'png', + INTERVAL '5' MINUTE) +FROM image_table; +``` + +Repeated short-term calls for the same descriptor reuse the materialized object and issue a fresh +URL. Treat the URL as a bearer credential: send it immediately to the consumer and never log or +persist it. Direct model `image_url` use is supported only for image formats verified with that +model; PDF is not covered. See [Blob Storage](../multimodal-table/blob#presigned-urls-for-oss-blobs) +for Java and Flink examples, caching behavior, and current limitations. + ## User-defined Function Paimon Spark supports three types of user-defined functions: lambda functions, file-based functions, and SQL functions. diff --git a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java index f11a056efbcf..e884b5664f53 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/Blob.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/Blob.java @@ -20,6 +20,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.utils.UriReader; @@ -28,6 +29,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.time.Duration; import java.util.function.Supplier; /** @@ -44,6 +46,11 @@ public interface Blob { SeekableInputStream newInputStream() throws IOException; + default String toPresignedUrl( + FileIO fileIO, Path tableRoot, String extension, Duration validity) throws IOException { + return fileIO.createBlobPresignedUrl(tableRoot, toDescriptor(), extension, validity); + } + static Blob fromData(byte[] data) { return new BlobData(data); } diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index 4315e9bcc4d2..10abaf19f79f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -20,6 +20,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.hadoop.HadoopFileIOLoader; import org.apache.paimon.fs.local.LocalFileIO; @@ -262,6 +263,13 @@ default Optional unarchive(Path path, StorageType type) throws IOException getClass().getName() + " does not support unarchive."); } + default String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + throw new UnsupportedOperationException( + getClass().getName() + " does not support creating blob presigned URLs."); + } + /** * Override this method to empty, many FileIO implementation classes rely on static variables * and do not have the ability to close them. diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java index d5b62a2944cf..e56ffd1d9287 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java @@ -19,9 +19,11 @@ package org.apache.paimon.fs; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.options.Options; import java.io.IOException; +import java.time.Duration; /** * A {@link FileIO} for plugin jar. {@link FileIO} is serializable, so plugin FileIO should be @@ -96,6 +98,17 @@ public boolean tryToWriteAtomic(Path path, String content) throws IOException { return wrap(() -> fileIO(path).tryToWriteAtomic(path, content)); } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + return wrap( + () -> + fileIO(new Path(descriptor.uri())) + .createBlobPresignedUrl( + tableRoot, descriptor, extension, validity)); + } + private FileIO fileIO(Path path) throws IOException { if (lazyFileIO == null) { synchronized (this) { diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java index 6a46d1a49008..699efb4d3cb1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java @@ -20,11 +20,13 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import java.io.IOException; import java.io.Serializable; +import java.time.Duration; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -107,6 +109,17 @@ public boolean rename(Path src, Path dst) throws IOException { return wrap(() -> fileIO(src).rename(src, dst)); } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + return wrap( + () -> + fileIO(new Path(descriptor.uri())) + .createBlobPresignedUrl( + tableRoot, descriptor, extension, validity)); + } + @VisibleForTesting public FileIO fileIO(Path path) throws IOException { CacheKey cacheKey = new CacheKey(path.toUri().getScheme(), path.toUri().getAuthority()); diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java index e41d0f04cde4..e403afab7108 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java @@ -19,6 +19,7 @@ package org.apache.paimon.fs.cache; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; @@ -32,6 +33,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.time.Duration; import java.util.EnumSet; import java.util.Map; import java.util.Set; @@ -151,6 +153,13 @@ public boolean rename(Path src, Path dst) throws IOException { return delegate.rename(src, dst); } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + return delegate.createBlobPresignedUrl(tableRoot, descriptor, extension, validity); + } + @Override public boolean isObjectStore() { return delegate.isObjectStore(); diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java index f9f0d283bbba..159d28c22608 100644 --- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; @@ -44,6 +45,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -150,6 +152,16 @@ public boolean rename(Path src, Path dst) throws IOException { return fileIO().rename(src, dst); } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + if (!path.equals(tableRoot)) { + throw new IOException("Table root does not match RESTTokenFileIO bound table root."); + } + return fileIO().createBlobPresignedUrl(tableRoot, descriptor, extension, validity); + } + @Override public boolean isObjectStore() { try { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java index 3c48f37fa329..1cc1366bc889 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/BlobDescriptorUtils.java @@ -20,18 +20,50 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URI; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import static org.apache.paimon.CoreOptions.BLOB_DESCRIPTOR_PREFIX; /** Utils for {@link BlobDescriptor}. */ public class BlobDescriptorUtils { + public static void validateTableRoot(Path tableRoot, BlobDescriptor descriptor) + throws IOException { + URI root = tableRoot.toUri().normalize(); + URI blob; + try { + blob = new Path(descriptor.uri()).toUri().normalize(); + } catch (RuntimeException e) { + throw new IOException("Invalid blob descriptor URI.", e); + } + + if (!Objects.equals(normalizeScheme(root.getScheme()), normalizeScheme(blob.getScheme())) + || !Objects.equals(root.getAuthority(), blob.getAuthority())) { + throw new IOException( + "Blob descriptor URI must use the same scheme and authority as table root."); + } + + String rootPath = root.getPath(); + String blobPath = blob.getPath(); + String childPrefix = rootPath.endsWith("/") ? rootPath : rootPath + "/"; + if (!blobPath.startsWith(childPrefix)) { + throw new IOException("Blob descriptor URI must be under table root."); + } + } + + private static String normalizeScheme(@Nullable String scheme) { + return scheme == null ? null : scheme.toLowerCase(); + } + /** * Try to create a {@link CatalogContext} for input {@link BlobDescriptor}. This enables reading * descriptors from external storages which can be different from paimon's own. diff --git a/paimon-common/src/test/java/org/apache/paimon/data/BlobTest.java b/paimon-common/src/test/java/org/apache/paimon/data/BlobTest.java index 50e34b29396c..b99b2b40a1a6 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/BlobTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobTest.java @@ -19,6 +19,8 @@ package org.apache.paimon.data; import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.utils.UriReader; @@ -30,11 +32,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.Duration; import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; /** Tests for {@link Blob}. */ public class BlobTest { @@ -85,6 +92,39 @@ public void testFromHttp() { assertThat(blob).isInstanceOf(BlobRef.class); } + @Test + public void testToPresignedUrlDelegatesDescriptor() throws IOException { + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 2, 4); + Blob blob = Blob.fromDescriptor(mock(UriReader.class), descriptor); + FileIO fileIO = mock(FileIO.class); + Path tableRoot = new Path("oss://bucket/table"); + Duration validity = Duration.ofHours(1); + when(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "jpg", validity)) + .thenReturn("https://example"); + + assertThat(blob.toPresignedUrl(fileIO, tableRoot, "jpg", validity)) + .isEqualTo("https://example"); + verify(fileIO).createBlobPresignedUrl(tableRoot, descriptor, "jpg", validity); + } + + @Test + public void testInlineBlobCannotCreatePresignedUrl() { + FileIO fileIO = mock(FileIO.class); + + assertThatThrownBy( + () -> + Blob.fromData(new byte[] {1}) + .toPresignedUrl( + fileIO, + new Path("oss://bucket/table"), + "jpg", + Duration.ofHours(1))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("can not convert to descriptor"); + verifyNoInteractions(fileIO); + } + @Test public void testBlobRefReadsKnownLengthDirectly() { byte[] data = new byte[8194]; diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/PluginFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/PluginFileIOTest.java new file mode 100644 index 000000000000..ba2fb2a9229b --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/fs/PluginFileIOTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +import org.apache.paimon.data.BlobDescriptor; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests for {@link PluginFileIO}. */ +class PluginFileIOTest { + + @Test + void testCreateBlobPresignedUrlUsesPluginClassLoader() throws IOException { + FileIO delegate = mock(FileIO.class); + ClassLoader pluginClassLoader = new ClassLoader() {}; + TestPluginFileIO fileIO = new TestPluginFileIO(delegate, pluginClassLoader); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 0, 1); + Duration validity = Duration.ofMinutes(5); + ClassLoader original = Thread.currentThread().getContextClassLoader(); + when(delegate.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .thenAnswer( + ignored -> { + assertThat(Thread.currentThread().getContextClassLoader()) + .isSameAs(pluginClassLoader); + return "https://example"; + }); + + assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .isEqualTo("https://example"); + assertThat(fileIO.createdFor).isEqualTo(new Path(descriptor.uri())); + assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(original); + } + + private static class TestPluginFileIO extends PluginFileIO { + + private final FileIO delegate; + private final ClassLoader classLoader; + private Path createdFor; + + private TestPluginFileIO(FileIO delegate, ClassLoader classLoader) { + this.delegate = delegate; + this.classLoader = classLoader; + } + + @Override + public boolean isObjectStore() { + return true; + } + + @Override + protected FileIO createFileIO(Path path) { + createdFor = path; + return delegate; + } + + @Override + protected ClassLoader pluginClassLoader() { + return classLoader; + } + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java index dae270eb873a..e5241c7cfbf4 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/ResolvingFileIOTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.fs; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.hadoop.HadoopFileIO; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; @@ -27,6 +28,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.time.Duration; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -34,6 +36,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Tests for {@link ResolvingFileIO}. */ public class ResolvingFileIOTest { @@ -137,4 +143,26 @@ public void testFileIOMapStoresFileIOInstances() throws IOException { assertEquals(localFileIO, localFileIOAgain); assertEquals(hdfsFileIO, hdfsFileIOAgain); } + + @Test + public void testCreateBlobPresignedUrlResolvesDescriptorFileIO() throws IOException { + FileIO delegate = mock(FileIO.class); + when(delegate.exists(any())).thenReturn(true); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenReturn(delegate); + when(loader.getScheme()).thenReturn("oss"); + resolvingFileIO.configure(CatalogContext.create(new Options(), loader, null)); + + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 0, 1); + Duration validity = Duration.ofMinutes(5); + when(delegate.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .thenReturn("https://example"); + + assertEquals( + "https://example", + resolvingFileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)); + verify(delegate).createBlobPresignedUrl(tableRoot, descriptor, "png", validity); + } } diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index bb179ed47baf..e4137be1aeca 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.fs.cache; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; @@ -31,11 +32,15 @@ import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.time.Duration; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Tests for {@link CachingFileIO} and {@link CachingSeekableInputStream}. */ class CachingFileIOTest { @@ -49,6 +54,27 @@ void setUp() { cacheDir = tempDir.resolve("cache").toString(); } + @Test + void testCreateBlobPresignedUrlDelegates() throws IOException { + FileIO delegate = mock(FileIO.class); + CachingFileIO cachingIO = + newCachingFileIO( + delegate, + new LocalMemoryCacheManager(1024, 64), + EnumSet.of(FileType.DATA), + 64); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 0, 1); + Duration validity = Duration.ofMinutes(5); + when(delegate.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .thenReturn("https://example"); + + assertThat(cachingIO.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .isEqualTo("https://example"); + verify(delegate).createBlobPresignedUrl(tableRoot, descriptor, "png", validity); + } + private CachingFileIO newCachingFileIO( FileIO delegate, LocalCacheManager cache, EnumSet whitelist, int blockSize) { return new CachingFileIO(delegate, cache, whitelist); diff --git a/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java new file mode 100644 index 000000000000..e4cce6608e3f --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/rest/RESTTokenFileIOTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileIOLoader; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.responses.GetTableTokenResponse; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for {@link RESTTokenFileIO}. */ +class RESTTokenFileIOTest { + + @Test + void testCreateBlobPresignedUrlRequiresBoundRootAndDelegates() throws IOException { + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 0, 1); + Duration validity = Duration.ofMinutes(5); + FileIO delegate = mock(FileIO.class); + when(delegate.exists(any())).thenReturn(true); + when(delegate.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .thenReturn("https://example"); + FileIOLoader loader = mock(FileIOLoader.class); + when(loader.load(any())).thenReturn(delegate); + when(loader.getScheme()).thenReturn("oss"); + RESTApi api = mock(RESTApi.class); + Identifier identifier = Identifier.create("db", "table"); + when(api.loadTableToken(identifier)) + .thenReturn(new GetTableTokenResponse(Collections.emptyMap(), Long.MAX_VALUE)); + RESTTokenFileIO fileIO = + new RESTTokenFileIO( + CatalogContext.create(new Options(), loader, null), + api, + identifier, + tableRoot); + + assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", validity)) + .isEqualTo("https://example"); + verify(delegate).createBlobPresignedUrl(tableRoot, descriptor, "png", validity); + + assertThatThrownBy( + () -> + fileIO.createBlobPresignedUrl( + new Path("oss://bucket/other"), + descriptor, + "png", + validity)) + .isInstanceOf(IOException.class) + .hasMessageContaining("bound table root"); + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/BlobDescriptorUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/BlobDescriptorUtilsTest.java new file mode 100644 index 000000000000..d970a289eb5a --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/utils/BlobDescriptorUtilsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link BlobDescriptorUtils}. */ +class BlobDescriptorUtilsTest { + + @Test + void testValidateTableRoot() { + Path tableRoot = new Path("oss://bucket/table"); + + assertThatCode( + () -> + BlobDescriptorUtils.validateTableRoot( + tableRoot, + new BlobDescriptor( + "oss://bucket/table/bucket-0/data.blob", 0, 1))) + .doesNotThrowAnyException(); + + assertInvalid(tableRoot, "oss://bucket/table-copy/data.blob"); + assertInvalid(tableRoot, "oss://bucket/table/../other/data.blob"); + assertInvalid(tableRoot, "oss://other/table/data.blob"); + assertInvalid(tableRoot, "s3://bucket/table/data.blob"); + } + + private static void assertInvalid(Path tableRoot, String uri) { + assertThatThrownBy( + () -> + BlobDescriptorUtils.validateTableRoot( + tableRoot, new BlobDescriptor(uri, 0, 1))) + .isInstanceOf(IOException.class); + } +} diff --git a/paimon-filesystems/paimon-oss-impl/pom.xml b/paimon-filesystems/paimon-oss-impl/pom.xml index 4083a1cdc018..ff089cd2e0bf 100644 --- a/paimon-filesystems/paimon-oss-impl/pom.xml +++ b/paimon-filesystems/paimon-oss-impl/pom.xml @@ -94,6 +94,13 @@ provided + + + org.mockito + mockito-core + ${mockito.version} + test + diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java index c397eca2f3de..a0d02e3f7a4f 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java @@ -19,16 +19,19 @@ package org.apache.paimon.oss; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.HadoopOptionsProvider; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.options.Options; +import org.apache.paimon.utils.BlobDescriptorUtils; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.ReflectionUtils; import org.apache.paimon.utils.SensitiveConfigUtils; import org.apache.paimon.utils.StringUtils; +import com.aliyun.oss.HttpMethod; import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSException; import com.aliyun.oss.common.auth.CredentialsProvider; @@ -36,17 +39,23 @@ import com.aliyun.oss.internal.OSSHeaders; import com.aliyun.oss.internal.OSSMultipartOperation; import com.aliyun.oss.internal.OSSObjectOperation; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; import com.aliyun.oss.model.CopyObjectRequest; import com.aliyun.oss.model.CopyObjectResult; import com.aliyun.oss.model.InitiateMultipartUploadRequest; import com.aliyun.oss.model.InitiateMultipartUploadResult; import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PartETag; import com.aliyun.oss.model.PutObjectRequest; import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore; +import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +63,18 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; +import java.net.URL; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -81,6 +100,9 @@ public class OSSFileIO extends HadoopCompliantFileIO implements HadoopOptionsPro private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; private static final String OSS_SECOND_LEVEL_DOMAIN_ENABLED = "fs.oss.sld.enabled"; + private static final String BLOB_FINGERPRINT_METADATA = "paimon-blob-descriptor-sha256"; + private static final long BLOB_COPY_MIN_PART_SIZE = 100L * 1024 * 1024; + private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); // Paimon OSS SSE keys, mapping 1:1 to the OSS headers; they take precedence over hadoop's // server-side-encryption-algorithm. /** SSE method -> x-oss-server-side-encryption (AES256 / KMS / SM4). */ @@ -255,6 +277,213 @@ public boolean tryToWriteAtomic(Path path, String content) throws IOException { } } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + BlobDescriptorUtils.validateTableRoot(tableRoot, descriptor); + if (validity == null + || validity.isZero() + || validity.isNegative() + || validity.getNano() != 0) { + throw new IOException("Blob presigned URL validity must be positive whole seconds."); + } + + String normalizedExtension = normalizeExtension(extension); + String contentType = contentType(normalizedExtension); + try { + URI source = new Path(descriptor.uri()).toUri(); + String bucket = source.getAuthority(); + String sourceKey = objectKey(source); + String fingerprint = sha256Hex(descriptor.serialize()); + int parentEnd = sourceKey.lastIndexOf('/') + 1; + String targetKey = + sourceKey.substring(0, parentEnd) + + "_bloburl_" + + fingerprint + + "." + + normalizedExtension; + OSSClient client = ossClient(new Path(descriptor.uri())); + + ObjectMetadata target = headObjectIfExists(client, bucket, targetKey); + if (!matches(target, descriptor.length(), contentType, fingerprint)) { + ObjectMetadata sourceMetadata = client.headObject(bucket, sourceKey); + validateRange(descriptor, sourceMetadata.getContentLength()); + materialize( + client, bucket, sourceKey, targetKey, descriptor, contentType, fingerprint); + target = client.headObject(bucket, targetKey); + if (!matches(target, descriptor.length(), contentType, fingerprint)) { + throw new IOException( + "Materialized blob object metadata does not match descriptor."); + } + } + + URL url = + client.generatePresignedUrl( + bucket, + targetKey, + Date.from(Instant.now().plus(validity)), + HttpMethod.GET); + validatePresignedUrl(client, url, bucket, targetKey); + return url.toString(); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Failed to create blob presigned URL.", e); + } + } + + private static void materialize( + OSSClient client, + String bucket, + String sourceKey, + String targetKey, + BlobDescriptor descriptor, + String contentType, + String fingerprint) + throws Exception { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentType(contentType); + metadata.addUserMetadata(BLOB_FINGERPRINT_METADATA, fingerprint); + if (descriptor.length() == 0) { + metadata.setContentLength(0); + client.putObject(bucket, targetKey, new ByteArrayInputStream(new byte[0]), metadata); + return; + } + + String uploadId = null; + try { + InitiateMultipartUploadResult initiated = + client.initiateMultipartUpload( + new InitiateMultipartUploadRequest(bucket, targetKey, metadata)); + uploadId = initiated.getUploadId(); + long partSize = + AliyunOSSUtils.calculatePartSize(descriptor.length(), BLOB_COPY_MIN_PART_SIZE); + List parts = new ArrayList<>(); + long copied = 0; + int partNumber = 1; + while (copied < descriptor.length()) { + long size = Math.min(partSize, descriptor.length() - copied); + UploadPartCopyResult result = + client.uploadPartCopy( + new UploadPartCopyRequest( + bucket, + sourceKey, + bucket, + targetKey, + uploadId, + partNumber, + descriptor.offset() + copied, + size)); + parts.add(new PartETag(partNumber, result.getETag())); + copied += size; + partNumber++; + } + client.completeMultipartUpload( + new CompleteMultipartUploadRequest(bucket, targetKey, uploadId, parts)); + uploadId = null; + } catch (Exception e) { + if (uploadId != null) { + try { + client.abortMultipartUpload( + new AbortMultipartUploadRequest(bucket, targetKey, uploadId)); + } catch (Exception abortException) { + e.addSuppressed(abortException); + } + } + throw e; + } + } + + private static ObjectMetadata headObjectIfExists(OSSClient client, String bucket, String key) { + try { + return client.headObject(bucket, key); + } catch (OSSException e) { + if ("NoSuchKey".equals(e.getErrorCode()) || "NoSuchObject".equals(e.getErrorCode())) { + return null; + } + throw e; + } + } + + private static boolean matches( + ObjectMetadata metadata, long length, String contentType, String fingerprint) { + return metadata != null + && metadata.getContentLength() == length + && contentType.equals(metadata.getContentType()) + && fingerprint.equals(metadata.getUserMetadata().get(BLOB_FINGERPRINT_METADATA)); + } + + private static void validateRange(BlobDescriptor descriptor, long sourceLength) + throws IOException { + if (descriptor.offset() < 0 + || descriptor.length() < 0 + || descriptor.offset() > sourceLength + || descriptor.length() > sourceLength - descriptor.offset()) { + throw new IOException("Blob descriptor range is outside the source object."); + } + } + + private static String normalizeExtension(String extension) throws IOException { + if (extension == null || extension.isEmpty()) { + throw new IOException("Blob file extension must contain only ASCII letters or digits."); + } + for (int i = 0; i < extension.length(); i++) { + char c = extension.charAt(i); + if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9')) { + throw new IOException( + "Blob file extension must contain only ASCII letters or digits."); + } + } + return extension.toLowerCase(Locale.ROOT); + } + + private static String contentType(String extension) throws IOException { + String contentType = URLConnection.guessContentTypeFromName("file." + extension); + if (contentType == null || "application/octet-stream".equals(contentType)) { + throw new IOException("Unknown MIME type for blob file extension: " + extension); + } + return contentType; + } + + private static String objectKey(URI uri) throws IOException { + String path = uri.getPath(); + if (path == null || !path.startsWith("/") || path.length() == 1) { + throw new IOException("Blob descriptor URI must contain an OSS object key."); + } + return path.substring(1); + } + + private static String sha256Hex(byte[] bytes) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(bytes); + char[] chars = new char[hash.length * 2]; + for (int i = 0; i < hash.length; i++) { + chars[i * 2] = HEX_CHARS[(hash[i] >> 4) & 0x0f]; + chars[i * 2 + 1] = HEX_CHARS[hash[i] & 0x0f]; + } + return new String(chars); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 not available.", e); + } + } + + private static void validatePresignedUrl( + OSSClient client, URL url, String bucket, String targetKey) throws Exception { + URI endpoint = client.getEndpoint(); + String expectedHost = bucket + "." + endpoint.getHost(); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) + || !"https".equalsIgnoreCase(url.getProtocol()) + || !expectedHost.equalsIgnoreCase(url.getHost()) + || !("/" + targetKey).equals(url.toURI().getPath())) { + throw new IOException("OSS client generated a presigned URL for an invalid target."); + } + } + + OSSClient ossClient(Path path) throws Exception { + return getOssClient((AliyunOSSFileSystem) getFileSystem(path(path))); + } + @Override public void close() { if (!allowCache) { diff --git a/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java b/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java index d6b642a024fa..3aea4df1ca5b 100644 --- a/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java +++ b/paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OSSFileIOTest.java @@ -18,31 +18,422 @@ package org.apache.paimon.oss; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; + import com.aliyun.oss.ClientConfiguration; import com.aliyun.oss.ClientException; +import com.aliyun.oss.HttpMethod; import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.common.comm.ExecutionContext; import com.aliyun.oss.common.comm.RequestMessage; import com.aliyun.oss.common.comm.ResponseMessage; import com.aliyun.oss.common.comm.RetryStrategy; import com.aliyun.oss.common.comm.ServiceClient; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; import com.aliyun.oss.model.CopyObjectRequest; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URL; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Tests for {@link OSSFileIO}. */ public class OSSFileIOTest { + @Test + public void testCreateBlobPresignedUrlMaterializesSinglePart() throws Exception { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/source.blob", 10, 20); + String fingerprint = sha256Hex(descriptor.serialize()); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(100); + ObjectMetadata target = new ObjectMetadata(); + target.setContentLength(20); + target.setContentType("image/png"); + target.addUserMetadata("paimon-blob-descriptor-sha256", fingerprint); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenThrow(missingObject()) + .thenReturn(target); + when(client.headObject("bucket", "table/bucket-0/source.blob")).thenReturn(source); + InitiateMultipartUploadResult initiate = new InitiateMultipartUploadResult(); + initiate.setUploadId("upload-id"); + when(client.initiateMultipartUpload(any())).thenReturn(initiate); + UploadPartCopyResult copied = new UploadPartCopyResult(); + copied.setPartNumber(1); + copied.setETag("etag"); + when(client.uploadPartCopy(any())).thenReturn(copied); + when(client.getEndpoint()).thenReturn(URI.create("https://oss-cn-hangzhou.aliyuncs.com")); + when(client.generatePresignedUrl(eq("bucket"), anyString(), any(), eq(HttpMethod.GET))) + .thenAnswer( + invocation -> + presignedUrl( + "https://bucket.oss-cn-hangzhou.aliyuncs.com/" + + invocation.getArgument(1) + + "?Signature=test")); + + String url = + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "PNG", + Duration.ofMinutes(5)); + + assertThat(url) + .matches( + "https://bucket\\.oss-cn-hangzhou\\.aliyuncs\\.com/" + + "table/bucket-0/_bloburl_[0-9a-f]{64}\\.png\\?Signature=test"); + ArgumentCaptor initiateCaptor = + ArgumentCaptor.forClass(InitiateMultipartUploadRequest.class); + verify(client).initiateMultipartUpload(initiateCaptor.capture()); + assertThat(initiateCaptor.getValue().getObjectMetadata().getContentType()) + .isEqualTo("image/png"); + assertThat(initiateCaptor.getValue().getObjectMetadata().getUserMetadata()) + .containsEntry("paimon-blob-descriptor-sha256", fingerprint); + + ArgumentCaptor copyCaptor = + ArgumentCaptor.forClass(UploadPartCopyRequest.class); + verify(client).uploadPartCopy(copyCaptor.capture()); + UploadPartCopyRequest copy = copyCaptor.getValue(); + assertThat(copy.getSourceBucketName()).isEqualTo("bucket"); + assertThat(copy.getSourceKey()).isEqualTo("table/bucket-0/source.blob"); + assertThat(copy.getBeginIndex()).isEqualTo(10); + assertThat(copy.getPartSize()).isEqualTo(20); + assertThat(copy.getPartNumber()).isEqualTo(1); + verify(client).completeMultipartUpload(any(CompleteMultipartUploadRequest.class)); + } + + @Test + public void testCreateBlobPresignedUrlCacheHitOnlyPresigns() throws Exception { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/bucket-0/source.blob", 10, 20); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenReturn(matchingMetadata(descriptor, "image/jpeg")); + stubPresigning(client); + + String url = + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "jpg", + Duration.ofMinutes(5)); + + assertThat(url).startsWith("https://bucket.oss-cn-hangzhou.aliyuncs.com/"); + verify(client, never()).headObject("bucket", "table/bucket-0/source.blob"); + verify(client, never()).initiateMultipartUpload(any()); + verify(client, never()).uploadPartCopy(any()); + } + + @Test + public void testCreateBlobPresignedUrlControlsMultipartCount() throws Exception { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/source.blob", 0, Long.MAX_VALUE); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(Long.MAX_VALUE); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenThrow(missingObject()) + .thenReturn(matchingMetadata(descriptor, "image/png")); + when(client.headObject("bucket", "table/source.blob")).thenReturn(source); + stubMultipartUpload(client); + stubPresigning(client); + + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), descriptor, "png", Duration.ofMinutes(5)); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(UploadPartCopyRequest.class); + verify(client, times(10_000)).uploadPartCopy(captor.capture()); + assertThat(captor.getAllValues().get(0).getBeginIndex()).isZero(); + assertThat(captor.getAllValues().get(9_999).getPartNumber()).isEqualTo(10_000); + } + + @Test + public void testCreateBlobPresignedUrlHandlesZeroLength() throws Exception { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/source.blob", 10, 0); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(10); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenThrow(missingObject()) + .thenReturn(matchingMetadata(descriptor, "image/png")); + when(client.headObject("bucket", "table/source.blob")).thenReturn(source); + stubPresigning(client); + + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), descriptor, "png", Duration.ofMinutes(5)); + + verify(client) + .putObject( + eq("bucket"), + contains("_bloburl_"), + any(java.io.InputStream.class), + any(ObjectMetadata.class)); + verify(client, never()).initiateMultipartUpload(any()); + } + + @Test + public void testCreateBlobPresignedUrlAbortsFailedUpload() { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/source.blob", 0, 20); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(20); + when(client.headObject(eq("bucket"), contains("_bloburl_"))).thenThrow(missingObject()); + when(client.headObject("bucket", "table/source.blob")).thenReturn(source); + InitiateMultipartUploadResult initiate = new InitiateMultipartUploadResult(); + initiate.setUploadId("upload-id"); + when(client.initiateMultipartUpload(any())).thenReturn(initiate); + when(client.uploadPartCopy(any())).thenThrow(new ClientException("failed")); + + assertThatThrownBy( + () -> + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "png", + Duration.ofMinutes(5))) + .isInstanceOf(java.io.IOException.class); + ArgumentCaptor captor = + ArgumentCaptor.forClass(AbortMultipartUploadRequest.class); + verify(client).abortMultipartUpload(captor.capture()); + assertThat(captor.getValue().getUploadId()).isEqualTo("upload-id"); + } + + @Test + public void testCreateBlobPresignedUrlRejectsFinalHeadMismatch() { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/source.blob", 0, 20); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(20); + ObjectMetadata wrongTarget = matchingMetadata(descriptor, "image/png"); + wrongTarget.setContentLength(19); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenThrow(missingObject()) + .thenReturn(wrongTarget); + when(client.headObject("bucket", "table/source.blob")).thenReturn(source); + stubMultipartUpload(client); + + assertThatThrownBy( + () -> + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "png", + Duration.ofMinutes(5))) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("metadata does not match"); + verify(client, never()) + .generatePresignedUrl(anyString(), anyString(), any(), any(HttpMethod.class)); + } + + @Test + public void testCreateBlobPresignedUrlRejectsInvalidArguments() { + OSSClient client = mock(OSSClient.class); + TestOSSFileIO fileIO = new TestOSSFileIO(client); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/source.blob", 0, 1); + Path root = new Path("oss://bucket/table"); + + assertThatThrownBy( + () -> + fileIO.createBlobPresignedUrl( + root, descriptor, ".png", Duration.ofSeconds(1))) + .isInstanceOf(java.io.IOException.class); + assertThatThrownBy( + () -> + fileIO.createBlobPresignedUrl( + root, descriptor, "unknown", Duration.ofSeconds(1))) + .isInstanceOf(java.io.IOException.class); + assertThatThrownBy( + () -> + fileIO.createBlobPresignedUrl( + root, descriptor, "png", Duration.ofMillis(1))) + .isInstanceOf(java.io.IOException.class); + assertThatThrownBy( + () -> + fileIO.createBlobPresignedUrl( + new Path("oss://bucket/other"), + descriptor, + "png", + Duration.ofSeconds(1))) + .isInstanceOf(java.io.IOException.class); + } + + @Test + public void testCreateBlobPresignedUrlRejectsInvalidRange() { + assertInvalidRange(-1, 1, 10); + assertInvalidRange(0, -1, 10); + assertInvalidRange(9, 2, 10); + assertInvalidRange(Long.MAX_VALUE, 1, Long.MAX_VALUE); + } + + @Test + public void testCreateBlobPresignedUrlRejectsInvalidTarget() { + assertInvalidPresignedUrl( + "https://oss-cn-hangzhou.aliyuncs.com", + "http://bucket.oss-cn-hangzhou.aliyuncs.com/table/_bloburl_hash.png"); + assertInvalidPresignedUrl( + "https://oss-cn-hangzhou.aliyuncs.com", + "https://other.oss-cn-hangzhou.aliyuncs.com/table/_bloburl_hash.png"); + assertInvalidPresignedUrl( + "https://oss-cn-hangzhou.aliyuncs.com", + "https://bucket.oss-cn-hangzhou.aliyuncs.com/table/other.png"); + assertInvalidPresignedUrl( + "http://oss-cn-hangzhou.aliyuncs.com", + "https://bucket.oss-cn-hangzhou.aliyuncs.com/table/_bloburl_hash.png"); + } + + private static void assertInvalidPresignedUrl(String endpoint, String generatedUrl) { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/source.blob", 0, 1); + when(client.headObject(eq("bucket"), contains("_bloburl_"))) + .thenReturn(matchingMetadata(descriptor, "image/png")); + when(client.getEndpoint()).thenReturn(URI.create(endpoint)); + when(client.generatePresignedUrl(eq("bucket"), anyString(), any(), eq(HttpMethod.GET))) + .thenReturn(presignedUrl(generatedUrl)); + + assertThatThrownBy( + () -> + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "png", + Duration.ofMinutes(5))) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("invalid target"); + } + + private static void assertInvalidRange(long offset, long length, long sourceLength) { + OSSClient client = mock(OSSClient.class); + BlobDescriptor descriptor = + new BlobDescriptor("oss://bucket/table/source.blob", offset, length); + ObjectMetadata source = new ObjectMetadata(); + source.setContentLength(sourceLength); + when(client.headObject(eq("bucket"), contains("_bloburl_"))).thenThrow(missingObject()); + when(client.headObject("bucket", "table/source.blob")).thenReturn(source); + + assertThatThrownBy( + () -> + new TestOSSFileIO(client) + .createBlobPresignedUrl( + new Path("oss://bucket/table"), + descriptor, + "png", + Duration.ofMinutes(5))) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("range"); + verify(client, never()).initiateMultipartUpload(any()); + } + + private static ObjectMetadata matchingMetadata(BlobDescriptor descriptor, String contentType) { + try { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(descriptor.length()); + metadata.setContentType(contentType); + metadata.addUserMetadata( + "paimon-blob-descriptor-sha256", sha256Hex(descriptor.serialize())); + return metadata; + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private static void stubMultipartUpload(OSSClient client) { + InitiateMultipartUploadResult initiate = new InitiateMultipartUploadResult(); + initiate.setUploadId("upload-id"); + when(client.initiateMultipartUpload(any())).thenReturn(initiate); + when(client.uploadPartCopy(any())) + .thenAnswer( + invocation -> { + UploadPartCopyRequest request = invocation.getArgument(0); + UploadPartCopyResult copied = new UploadPartCopyResult(); + copied.setPartNumber(request.getPartNumber()); + copied.setETag("etag-" + request.getPartNumber()); + return copied; + }); + } + + private static void stubPresigning(OSSClient client) { + when(client.getEndpoint()).thenReturn(URI.create("https://oss-cn-hangzhou.aliyuncs.com")); + when(client.generatePresignedUrl(eq("bucket"), anyString(), any(), eq(HttpMethod.GET))) + .thenAnswer( + invocation -> + presignedUrl( + "https://bucket.oss-cn-hangzhou.aliyuncs.com/" + + invocation.getArgument(1) + + "?Signature=test")); + } + + private static OSSException missingObject() { + return new OSSException("missing", "NoSuchKey", "request", "host", null, null, "HEAD"); + } + + private static URL presignedUrl(String url) { + try { + return new URL(url); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static String sha256Hex(byte[] bytes) throws NoSuchAlgorithmException { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(bytes); + StringBuilder result = new StringBuilder(hash.length * 2); + for (byte value : hash) { + result.append(String.format("%02x", value & 0xff)); + } + return result.toString(); + } + + private static class TestOSSFileIO extends OSSFileIO { + + private final OSSClient client; + + private TestOSSFileIO(OSSClient client) { + this.client = client; + } + + OSSClient ossClient(Path path) { + return client; + } + } + /** The swap must replace both operations with the SSE-stamping subclasses. */ @Test public void testSseOperationSwapTakesEffect() throws Exception { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/BuiltInFunctions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/BuiltInFunctions.java index 7ab2a90f1f3a..cada89b4ec14 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/BuiltInFunctions.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/BuiltInFunctions.java @@ -30,6 +30,12 @@ public class BuiltInFunctions { put("path_to_descriptor", PathToDescriptor.class.getName()); put("descriptor_to_string", DescriptorToString.class.getName()); put("blob_view", BlobViewFunction.class.getName()); + put( + "descriptor_to_presigned_url", + DescriptorToPresignedUrlFunction.class.getName()); + put( + "try_descriptor_to_presigned_url", + TryDescriptorToPresignedUrlFunction.class.getName()); } }; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunction.java new file mode 100644 index 000000000000..83b5bd4b4443 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunction.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.function; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.flink.FlinkFileIOLoader; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.Table; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.types.inference.InputTypeStrategies; +import org.apache.flink.table.types.inference.TypeInference; +import org.apache.flink.table.types.inference.TypeStrategies; +import org.apache.flink.table.types.logical.LogicalTypeRoot; + +import javax.annotation.Nullable; + +import java.time.Duration; + +/** Converts a serialized blob descriptor to a presigned URL. */ +public class DescriptorToPresignedUrlFunction extends ScalarFunction + implements CatalogAwareFunction { + + @Nullable private String catalogName; + @Nullable private Options catalogOptions; + + private transient Catalog catalog; + private transient String sourceTable; + private transient FileIO fileIO; + private transient Path tableRoot; + + public DescriptorToPresignedUrlFunction() {} + + DescriptorToPresignedUrlFunction(String catalogName, Catalog catalog) { + this.catalogName = catalogName; + this.catalog = catalog; + } + + @Override + public void setCatalogContext( + String catalogName, String defaultDatabase, Options catalogOptions) { + this.catalogName = catalogName; + this.catalogOptions = new Options(catalogOptions.toMap()); + this.catalog = null; + this.sourceTable = null; + this.fileIO = null; + this.tableRoot = null; + } + + @Override + public void open(FunctionContext context) { + openCatalog(context.getUserCodeClassLoader()); + } + + @Override + public void close() throws Exception { + if (catalog != null) { + catalog.close(); + catalog = null; + } + } + + public String eval( + String tableName, byte[] descriptorBytes, String extension, Duration validity) + throws Exception { + bindSourceTable(tableName); + if (descriptorBytes == null || extension == null || validity == null) { + return null; + } + + loadTable(); + return fileIO.createBlobPresignedUrl( + tableRoot, BlobDescriptor.deserialize(descriptorBytes), extension, validity); + } + + @Override + public TypeInference getTypeInference(DataTypeFactory typeFactory) { + return TypeInference.newBuilder() + .inputTypeStrategy( + InputTypeStrategies.sequence( + InputTypeStrategies.and( + InputTypeStrategies.explicit(DataTypes.STRING()), + InputTypeStrategies.LITERAL), + InputTypeStrategies.explicit(DataTypes.BYTES()), + InputTypeStrategies.explicit(DataTypes.STRING()), + InputTypeStrategies.logical(LogicalTypeRoot.INTERVAL_DAY_TIME))) + .outputTypeStrategy(TypeStrategies.explicit(DataTypes.STRING())) + .build(); + } + + @Override + public boolean isDeterministic() { + return false; + } + + private void bindSourceTable(String tableName) { + if (tableName == null) { + throw new IllegalArgumentException("source_table must be a non-null literal."); + } + if (sourceTable == null) { + sourceTable = tableName; + } else if (!sourceTable.equals(tableName)) { + throw new IllegalArgumentException( + "A descriptor URL function instance can use only one source table."); + } + } + + private void loadTable() throws Catalog.TableNotExistException { + if (fileIO != null) { + return; + } + + Identifier identifier = toIdentifier(sourceTable); + Table table = catalog().getTable(identifier); + if (!(table instanceof DataTable)) { + throw new IllegalArgumentException( + "Source table " + identifier.getFullName() + " is not a data table."); + } + DataTable dataTable = (DataTable) table; + fileIO = dataTable.fileIO(); + tableRoot = dataTable.location(); + } + + private Identifier toIdentifier(String tableName) { + String[] parts = tableName.split("\\.", -1); + if (parts.length == 2 && !parts[0].isEmpty() && !parts[1].isEmpty()) { + return new Identifier(parts[0], parts[1]); + } + if (parts.length == 3 + && catalogName != null + && catalogName.equals(parts[0]) + && !parts[1].isEmpty() + && !parts[2].isEmpty()) { + return new Identifier(parts[1], parts[2]); + } + throw new IllegalArgumentException( + "source_table must be 'database.table' or '" + catalogName + ".database.table'."); + } + + private Catalog catalog() { + if (catalog == null) { + openCatalog(Thread.currentThread().getContextClassLoader()); + } + return catalog; + } + + private void openCatalog(ClassLoader classLoader) { + if (catalog != null) { + return; + } + if (catalogOptions == null) { + throw new IllegalStateException( + "DescriptorToPresignedUrlFunction is missing catalog options."); + } + catalog = + CatalogFactory.createCatalog( + CatalogContext.create(catalogOptions, new FlinkFileIOLoader()), + classLoader); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/TryDescriptorToPresignedUrlFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/TryDescriptorToPresignedUrlFunction.java new file mode 100644 index 000000000000..33d4589eb65d --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/function/TryDescriptorToPresignedUrlFunction.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.function; + +import org.apache.paimon.catalog.Catalog; + +import java.time.Duration; + +/** Tolerant variant of {@link DescriptorToPresignedUrlFunction}. */ +public class TryDescriptorToPresignedUrlFunction extends DescriptorToPresignedUrlFunction { + + public TryDescriptorToPresignedUrlFunction() {} + + TryDescriptorToPresignedUrlFunction(String catalogName, Catalog catalog) { + super(catalogName, catalog); + } + + @Override + public String eval( + String tableName, byte[] descriptorBytes, String extension, Duration validity) { + try { + return super.eval(tableName, descriptorBytes, extension, validity); + } catch (Exception e) { + return null; + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java index 68db7adce82e..44eda01535e5 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BlobTableITCase.java @@ -622,6 +622,38 @@ public void testWriteBlobWithBuiltInFunction() throws Exception { .containsExactlyInAnyOrder(Row.of(1, "paimon", blobData)); } + @Test + public void testDescriptorToPresignedUrlBuiltInFunctions() { + assertThat(batchSql("SHOW FUNCTIONS IN sys")) + .contains( + Row.of("descriptor_to_presigned_url"), + Row.of("try_descriptor_to_presigned_url")); + + assertThat( + batchSql( + "SELECT sys.descriptor_to_presigned_url(" + + "'default.blob_table_descriptor', " + + "CAST(NULL AS BYTES), 'png', INTERVAL '1' HOUR)")) + .containsExactly(Row.of((Object) null)); + assertThat( + batchSql( + "SELECT sys.try_descriptor_to_presigned_url(" + + "'default.blob_table_descriptor', " + + "sys.path_to_descriptor('file:///tmp/blob'), " + + "'png', INTERVAL '1' HOUR)")) + .containsExactly(Row.of((Object) null)); + + assertThatThrownBy( + () -> + batchSql( + "SELECT sys.descriptor_to_presigned_url(" + + "'default.blob_table_descriptor', " + + "sys.path_to_descriptor('file:///tmp/blob'), " + + "'png', INTERVAL '1' HOUR)")) + .hasRootCauseInstanceOf(UnsupportedOperationException.class) + .hasStackTraceContaining("does not support creating blob presigned URLs"); + } + @Test public void testWriteBlobViewWithBuiltInFunction() throws Exception { tEnv.executeSql( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunctionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunctionTest.java new file mode 100644 index 000000000000..fda3abb25dfe --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/function/DescriptorToPresignedUrlFunctionTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.function; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.DataTable; + +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.ValidationException; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for descriptor to presigned URL functions. */ +public class DescriptorToPresignedUrlFunctionTest { + + @Test + public void testTypeInferenceRequiresLiteralTableAndAcceptsInterval() { + TableEnvironment tableEnvironment = + TableEnvironment.create(EnvironmentSettings.inBatchMode()); + tableEnvironment.createTemporarySystemFunction( + "descriptor_to_presigned_url", DescriptorToPresignedUrlFunction.class); + + assertThat( + tableEnvironment.explainSql( + "SELECT descriptor_to_presigned_url(" + + "'default.t', CAST(NULL AS BYTES), 'png', " + + "INTERVAL '1' HOUR)")) + .contains("Calc"); + + assertThatThrownBy( + () -> + tableEnvironment.explainSql( + "SELECT descriptor_to_presigned_url(" + + "source_table, descriptor, 'png', " + + "INTERVAL '1' HOUR) " + + "FROM (VALUES ('default.t', CAST(NULL AS BYTES))) " + + "AS T(source_table, descriptor)")) + .isInstanceOf(ValidationException.class) + .hasRootCauseMessage("Literal expected."); + } + + @Test + public void testEvalCachesOneTableAndDelegates() throws Exception { + Catalog catalog = mock(Catalog.class); + DataTable table = mock(DataTable.class); + FileIO fileIO = mock(FileIO.class); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/bucket-0/data", 1, 2); + when(catalog.getTable(new Identifier("default", "t"))).thenReturn(table); + when(table.location()).thenReturn(tableRoot); + when(table.fileIO()).thenReturn(fileIO); + when(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", Duration.ofHours(1))) + .thenReturn("https://example"); + + DescriptorToPresignedUrlFunction function = + new DescriptorToPresignedUrlFunction("paimon", catalog); + + assertThat(function.eval("default.t", descriptor.serialize(), "png", Duration.ofHours(1))) + .isEqualTo("https://example"); + assertThat(function.eval("default.t", descriptor.serialize(), "png", Duration.ofHours(1))) + .isEqualTo("https://example"); + verify(catalog, times(1)).getTable(new Identifier("default", "t")); + + assertThatThrownBy( + () -> + function.eval( + "default.other", + descriptor.serialize(), + "png", + Duration.ofHours(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("one source table"); + } + + @Test + public void testNullAndTrySemantics() throws Exception { + Catalog catalog = mock(Catalog.class); + DataTable table = mock(DataTable.class); + FileIO fileIO = mock(FileIO.class); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/data", 0, 1); + when(catalog.getTable(new Identifier("default", "t"))).thenReturn(table); + when(table.location()).thenReturn(tableRoot); + when(table.fileIO()).thenReturn(fileIO); + when(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", Duration.ofSeconds(1))) + .thenThrow(new IOException("signing failed")); + + DescriptorToPresignedUrlFunction strict = + new DescriptorToPresignedUrlFunction("paimon", catalog); + assertThat(strict.eval("default.t", null, "png", Duration.ofSeconds(1))).isNull(); + assertThat(strict.eval("default.t", descriptor.serialize(), null, Duration.ofSeconds(1))) + .isNull(); + assertThat(strict.eval("default.t", descriptor.serialize(), "png", null)).isNull(); + verify(catalog, never()).getTable(new Identifier("default", "t")); + assertThatThrownBy( + () -> + strict.eval( + "default.t", + descriptor.serialize(), + "png", + Duration.ofSeconds(1))) + .isInstanceOf(IOException.class) + .hasMessageContaining("signing failed"); + + TryDescriptorToPresignedUrlFunction tolerant = + new TryDescriptorToPresignedUrlFunction("paimon", catalog); + assertThat(tolerant.eval("default.t", descriptor.serialize(), "png", Duration.ofSeconds(1))) + .isNull(); + } + + @Test + public void testRegistrationAndDeterminism() { + assertThat(BuiltInFunctions.FUNCTIONS) + .containsEntry( + "descriptor_to_presigned_url", + DescriptorToPresignedUrlFunction.class.getName()) + .containsEntry( + "try_descriptor_to_presigned_url", + TryDescriptorToPresignedUrlFunction.class.getName()); + assertThat(new DescriptorToPresignedUrlFunction().isDeterministic()).isFalse(); + assertThat(new TryDescriptorToPresignedUrlFunction().isDeterministic()).isFalse(); + } + + @Test + public void testCatalogQualifiedTableName() throws Exception { + Catalog catalog = mock(Catalog.class); + DataTable table = mock(DataTable.class); + FileIO fileIO = mock(FileIO.class); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/data", 0, 1); + when(catalog.getTable(new Identifier("default", "t"))).thenReturn(table); + when(table.location()).thenReturn(tableRoot); + when(table.fileIO()).thenReturn(fileIO); + when(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "png", Duration.ofSeconds(1))) + .thenReturn("https://example"); + + DescriptorToPresignedUrlFunction function = + new DescriptorToPresignedUrlFunction("paimon", catalog); + assertThat( + function.eval( + "paimon.default.t", + descriptor.serialize(), + "png", + Duration.ofSeconds(1))) + .isEqualTo("https://example"); + + DescriptorToPresignedUrlFunction crossCatalog = + new DescriptorToPresignedUrlFunction("paimon", catalog); + assertThatThrownBy( + () -> + crossCatalog.eval( + "other.default.t", + descriptor.serialize(), + "png", + Duration.ofSeconds(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("paimon.database.table"); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 7b5261f7d7f7..4e18c2318a56 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -723,7 +723,7 @@ public Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceExce public UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionException { String[] namespace = ident.namespace(); if (isSystemFunctionNamespace(namespace)) { - UnboundFunction func = PaimonFunctions.load(ident.name()); + UnboundFunction func = PaimonFunctions.load(ident.name(), catalogName); if (func != null) { return func; } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunction.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunction.java new file mode 100644 index 000000000000..b278552ddfb8 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunction.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.function; + +import org.apache.spark.sql.connector.catalog.functions.ScalarFunction; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.DayTimeIntervalType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.io.Serializable; + +/** Spark scalar function placeholder replaced with a table-scoped function during analysis. */ +public class DescriptorToPresignedUrlFunction implements ScalarFunction, Serializable { + + private static final long serialVersionUID = 1L; + + private final boolean ignoreErrors; + private final String catalogName; + private final DayTimeIntervalType intervalType; + + public DescriptorToPresignedUrlFunction(boolean ignoreErrors) { + this(ignoreErrors, null, DataTypes.createDayTimeIntervalType()); + } + + public DescriptorToPresignedUrlFunction(boolean ignoreErrors, String catalogName) { + this(ignoreErrors, catalogName, DataTypes.createDayTimeIntervalType()); + } + + public DescriptorToPresignedUrlFunction( + boolean ignoreErrors, String catalogName, DayTimeIntervalType intervalType) { + this.ignoreErrors = ignoreErrors; + this.catalogName = catalogName; + this.intervalType = intervalType; + } + + @Override + public DataType[] inputTypes() { + return new DataType[] { + DataTypes.StringType, DataTypes.BinaryType, DataTypes.StringType, intervalType + }; + } + + @Override + public DataType resultType() { + return DataTypes.StringType; + } + + public UTF8String invoke( + UTF8String sourceTable, byte[] descriptor, UTF8String extension, long validityMicros) { + if (sourceTable == null || descriptor == null || extension == null) { + return null; + } + throw new UnsupportedOperationException( + "Function '" + name() + "' requires a non-null literal sourceTable."); + } + + @Override + public boolean isDeterministic() { + return false; + } + + public boolean ignoreErrors() { + return ignoreErrors; + } + + public String catalogName() { + return catalogName; + } + + @Override + public String name() { + return DescriptorToPresignedUrlUnbound.functionName(ignoreErrors); + } + + @Override + public String canonicalName() { + return "paimon." + name() + "(string, binary, string, interval day to second)"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlUnbound.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlUnbound.java new file mode 100644 index 000000000000..bbbd7d1d6031 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlUnbound.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.function; + +import org.apache.spark.sql.connector.catalog.functions.BoundFunction; +import org.apache.spark.sql.connector.catalog.functions.UnboundFunction; +import org.apache.spark.sql.types.BinaryType; +import org.apache.spark.sql.types.DayTimeIntervalType; +import org.apache.spark.sql.types.StringType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +/** Function unbound to {@link DescriptorToPresignedUrlFunction}. */ +public class DescriptorToPresignedUrlUnbound implements UnboundFunction { + + public static final String NAME = "descriptor_to_presigned_url"; + public static final String TRY_NAME = "try_descriptor_to_presigned_url"; + + private final boolean ignoreErrors; + private final String catalogName; + + public DescriptorToPresignedUrlUnbound(boolean ignoreErrors) { + this(ignoreErrors, null); + } + + public DescriptorToPresignedUrlUnbound(boolean ignoreErrors, String catalogName) { + this.ignoreErrors = ignoreErrors; + this.catalogName = catalogName; + } + + @Override + public BoundFunction bind(StructType inputType) { + StructField[] fields = inputType.fields(); + if (fields.length != 4) { + throw new UnsupportedOperationException( + "Function '" + + name() + + "' requires 4 arguments " + + "(sourceTable STRING, descriptor BINARY, extension STRING, " + + "validity INTERVAL DAY TO SECOND), but found " + + fields.length); + } + requireType(fields, 0, StringType.class, "first", "STRING"); + requireType(fields, 1, BinaryType.class, "second", "BINARY"); + requireType(fields, 2, StringType.class, "third", "STRING"); + requireType(fields, 3, DayTimeIntervalType.class, "fourth", "INTERVAL DAY TO SECOND"); + return new DescriptorToPresignedUrlFunction( + ignoreErrors, catalogName, (DayTimeIntervalType) fields[3].dataType()); + } + + private void requireType( + StructField[] fields, + int position, + Class expectedClass, + String positionName, + String expectedName) { + if (!expectedClass.isInstance(fields[position].dataType())) { + throw new UnsupportedOperationException( + "The " + + positionName + + " argument of '" + + name() + + "' must be " + + expectedName + + " type, but found " + + fields[position].dataType().simpleString()); + } + } + + @Override + public String description() { + return "Create a presigned URL for a blob descriptor"; + } + + @Override + public String name() { + return functionName(ignoreErrors); + } + + static String functionName(boolean ignoreErrors) { + return ignoreErrors ? TRY_NAME : NAME; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/ResolvedDescriptorToPresignedUrlFunction.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/ResolvedDescriptorToPresignedUrlFunction.java new file mode 100644 index 000000000000..0bdfdc8ba043 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/function/ResolvedDescriptorToPresignedUrlFunction.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.function; + +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.utils.ExceptionUtils; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.functions.ScalarFunction; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.unsafe.types.UTF8String; + +import java.io.IOException; +import java.io.Serializable; +import java.time.Duration; + +/** Executor-side descriptor-to-presigned-URL function bound to one table's FileIO. */ +public class ResolvedDescriptorToPresignedUrlFunction + implements ScalarFunction, Serializable { + + private static final long serialVersionUID = 1L; + private static final long MICROS_PER_SECOND = 1_000_000L; + + private final FileIO fileIO; + private final Path tableRoot; + private final boolean ignoreErrors; + + public ResolvedDescriptorToPresignedUrlFunction( + FileIO fileIO, Path tableRoot, boolean ignoreErrors) { + this.fileIO = fileIO; + this.tableRoot = tableRoot; + this.ignoreErrors = ignoreErrors; + } + + @Override + public DataType[] inputTypes() { + return new DataType[] { + DataTypes.BinaryType, DataTypes.StringType, DataTypes.createDayTimeIntervalType() + }; + } + + @Override + public DataType resultType() { + return DataTypes.StringType; + } + + public UTF8String invoke(byte[] descriptor, UTF8String extension, long validityMicros) + throws IOException { + if (descriptor == null || extension == null) { + return null; + } + + try { + Duration validity = validity(validityMicros); + BlobDescriptor blobDescriptor = BlobDescriptor.deserialize(descriptor); + return UTF8String.fromString( + fileIO.createBlobPresignedUrl( + tableRoot, blobDescriptor, extension.toString(), validity)); + } catch (Exception e) { + if (ignoreErrors) { + return null; + } + if (e instanceof IOException) { + throw (IOException) e; + } + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException(e); + } + } + + @Override + public UTF8String produceResult(InternalRow input) { + if (input.isNullAt(0) || input.isNullAt(1) || input.isNullAt(2)) { + return null; + } + try { + return invoke(input.getBinary(0), input.getUTF8String(1), input.getLong(2)); + } catch (IOException e) { + ExceptionUtils.rethrow(e); + return null; + } + } + + private Duration validity(long validityMicros) { + if (validityMicros <= 0) { + throw new IllegalArgumentException("Validity interval must be positive."); + } + if (validityMicros % MICROS_PER_SECOND != 0) { + throw new IllegalArgumentException("Validity interval must contain whole seconds."); + } + return Duration.ofSeconds(validityMicros / MICROS_PER_SECOND); + } + + @Override + public boolean isDeterministic() { + return false; + } + + @Override + public String name() { + return DescriptorToPresignedUrlUnbound.functionName(ignoreErrors); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalog/functions/PaimonFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalog/functions/PaimonFunctions.scala index 56f6f2378b27..0ac3b5f6bf95 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalog/functions/PaimonFunctions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalog/functions/PaimonFunctions.scala @@ -25,7 +25,7 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.{ImmutableMap, import org.apache.paimon.spark.SparkInternalRowWrapper import org.apache.paimon.spark.SparkTypeUtils.toPaimonRowType import org.apache.paimon.spark.catalog.functions.PaimonFunctions._ -import org.apache.paimon.spark.function.{BlobViewUnbound, DescriptorToStringUnbound, PathToDescriptorUnbound} +import org.apache.paimon.spark.function.{BlobViewUnbound, DescriptorToPresignedUrlUnbound, DescriptorToStringUnbound, PathToDescriptorUnbound} import org.apache.paimon.table.{BucketMode, FileStoreTable} import org.apache.paimon.types.{ArrayType, DataType => PaimonDataType, LocalZonedTimestampType, MapType, RowType, TimestampType} import org.apache.paimon.utils.ProjectedRow @@ -48,6 +48,8 @@ object PaimonFunctions { val PATH_TO_DESCRIPTOR: String = "path_to_descriptor" val DESCRIPTOR_TO_STRING: String = "descriptor_to_string" val BLOB_VIEW: String = "blob_view" + val DESCRIPTOR_TO_PRESIGNED_URL: String = "descriptor_to_presigned_url" + val TRY_DESCRIPTOR_TO_PRESIGNED_URL: String = "try_descriptor_to_presigned_url" private val FUNCTIONS = ImmutableMap .builder[String, UnboundFunction]() @@ -58,6 +60,8 @@ object PaimonFunctions { .put(PATH_TO_DESCRIPTOR, new PathToDescriptorUnbound) .put(DESCRIPTOR_TO_STRING, new DescriptorToStringUnbound) .put(BLOB_VIEW, new BlobViewUnbound) + .put(DESCRIPTOR_TO_PRESIGNED_URL, new DescriptorToPresignedUrlUnbound(false)) + .put(TRY_DESCRIPTOR_TO_PRESIGNED_URL, new DescriptorToPresignedUrlUnbound(true)) .build() /** The bucket function type to the function name mapping */ @@ -76,6 +80,17 @@ object PaimonFunctions { @Nullable def load(name: String): UnboundFunction = FUNCTIONS.get(name) + + @Nullable + def load(name: String, catalogName: String): UnboundFunction = { + if (name.equalsIgnoreCase(DESCRIPTOR_TO_PRESIGNED_URL)) { + new DescriptorToPresignedUrlUnbound(false, catalogName) + } else if (name.equalsIgnoreCase(TRY_DESCRIPTOR_TO_PRESIGNED_URL)) { + new DescriptorToPresignedUrlUnbound(true, catalogName) + } else { + load(name) + } + } } /** diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala index 7a87cf7e3192..2b05994c923c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonFunctionResolver.scala @@ -24,12 +24,13 @@ import org.apache.paimon.spark.catalog.functions.PaimonFunctions import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction -import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, Literal} import org.apache.spark.sql.catalyst.parser.extensions.UnResolvedPaimonV1Function import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.UNRESOLVED_FUNCTION -import org.apache.spark.sql.types.{LongType, StringType} +import org.apache.spark.sql.connector.catalog.CatalogPlugin +import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, LongType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] { @@ -40,6 +41,10 @@ case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] plan.resolveOperatorsUpWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) { case l: LogicalPlan => l.transformExpressionsWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) { + case u: UnresolvedFunction + if isDescriptorToPresignedUrlFunction(u.nameParts) && + u.arguments.forall(_.resolved) => + resolveDescriptorToPresignedUrl(u) case u: UnresolvedFunction if isBlobViewFunction(u.nameParts) && u.arguments.forall(_.resolved) => resolveBlobView(u) @@ -73,6 +78,67 @@ case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] ReplacePaimonFunctions.resolveBlobView(spark, tableName, fieldName, u.arguments(2)) } + private def resolveDescriptorToPresignedUrl(u: UnresolvedFunction): Expression = { + val functionName = u.nameParts.last + if (u.arguments.length != 4) { + throw new UnsupportedOperationException( + s"Function '$functionName' requires 4 arguments " + + s"(sourceTable STRING, descriptor BINARY, extension STRING, " + + s"validity INTERVAL DAY TO SECOND), but found ${u.arguments.length}") + } + + val sourceTable = literalString(u.arguments(0), "sourceTable") + if (sourceTable == null) { + throw new UnsupportedOperationException("sourceTable must not be null") + } + + val descriptor = castNullable(u.arguments(1), BinaryType, "descriptor") + val extension = castNullable(u.arguments(2), StringType, "extension") + val intervalType = DayTimeIntervalType.DEFAULT + val validity = u.arguments(3).dataType match { + case _: DayTimeIntervalType => Cast(u.arguments(3), intervalType) + case NullType => Cast(u.arguments(3), intervalType) + case other => + throw new UnsupportedOperationException( + s"validity must be INTERVAL DAY TO SECOND type, but found ${other.simpleString}") + } + val ignoreErrors = + functionName.equalsIgnoreCase(PaimonFunctions.TRY_DESCRIPTOR_TO_PRESIGNED_URL) + + ReplacePaimonFunctions.resolveDescriptorToPresignedUrl( + spark, + functionCatalog(u.nameParts), + sourceTable, + descriptor, + extension, + validity, + ignoreErrors) + } + + private def castNullable( + expression: Expression, + expectedType: DataType, + argumentName: String): Expression = { + expression.dataType match { + case NullType => Cast(expression, expectedType) + case actual if actual == expectedType => expression + case actual => + throw new UnsupportedOperationException( + s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"but found ${actual.simpleString}") + } + } + + private def functionCatalog(nameParts: Seq[String]): CatalogPlugin = { + nameParts.length match { + case 2 => catalogManager.currentCatalog + case 3 => catalogManager.catalog(nameParts.head) + case _ => + throw new UnsupportedOperationException( + s"Invalid function identifier: ${nameParts.mkString(".")}") + } + } + private def literalString(expr: Expression, argumentName: String): String = { if (!expr.isInstanceOf[Literal]) { throw new UnsupportedOperationException(s"$argumentName must be a literal") @@ -94,4 +160,11 @@ case class PaimonFunctionResolver(spark: SparkSession) extends Rule[LogicalPlan] nameParts.last.equalsIgnoreCase(PaimonFunctions.BLOB_VIEW) && nameParts(nameParts.length - 2).equalsIgnoreCase(SYSTEM_DATABASE_NAME) } + + private def isDescriptorToPresignedUrlFunction(nameParts: Seq[String]): Boolean = { + nameParts.length >= 2 && + (nameParts.last.equalsIgnoreCase(PaimonFunctions.DESCRIPTOR_TO_PRESIGNED_URL) || + nameParts.last.equalsIgnoreCase(PaimonFunctions.TRY_DESCRIPTOR_TO_PRESIGNED_URL)) && + nameParts(nameParts.length - 2).equalsIgnoreCase(SYSTEM_DATABASE_NAME) + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala index ff34fcb18051..0350eef9c562 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/ReplacePaimonFunctions.scala @@ -21,8 +21,9 @@ package org.apache.paimon.spark.catalyst.analysis import org.apache.paimon.spark.{DataConverter, SparkTable, SparkTypeUtils, SparkUtils} import org.apache.paimon.spark.catalog.SparkBaseCatalog import org.apache.paimon.spark.catalog.functions.PaimonFunctions -import org.apache.paimon.spark.function.{BlobViewFieldIdSparkFunction, BlobViewSparkFunction} +import org.apache.paimon.spark.function.{BlobViewFieldIdSparkFunction, BlobViewSparkFunction, DescriptorToPresignedUrlFunction, ResolvedDescriptorToPresignedUrlFunction} import org.apache.paimon.spark.utils.CatalogUtils +import org.apache.paimon.table.DataTable import org.apache.paimon.types.DataTypeRoot import org.apache.paimon.utils.{InternalRowUtils, TypeUtils} @@ -31,14 +32,59 @@ import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, Cast, import org.apache.spark.sql.catalyst.expressions.objects.Invoke import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier} import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ -import org.apache.spark.sql.types.{BinaryType, StringType} +import org.apache.spark.sql.types.{BinaryType, DataType, DayTimeIntervalType, NullType, StringType} import org.apache.spark.unsafe.types.UTF8String import scala.collection.JavaConverters._ object ReplacePaimonFunctions { + def resolveDescriptorToPresignedUrl( + spark: SparkSession, + functionCatalog: CatalogPlugin, + sourceTable: String, + descriptor: Expression, + extension: Expression, + validity: Expression, + ignoreErrors: Boolean): Expression = { + if (!functionCatalog.isInstanceOf[SparkBaseCatalog]) { + throw new UnsupportedOperationException(s"$functionCatalog is not a Paimon catalog") + } + + val parts = spark.sessionState.sqlParser.parseMultipartIdentifier(sourceTable) + val identifier = parts.length match { + case 2 => Identifier.of(Array(parts.head), parts(1)) + case 3 => + if (!parts.head.equalsIgnoreCase(functionCatalog.name())) { + throw new UnsupportedOperationException( + s"Source table catalog '${parts.head}' must match function catalog " + + s"'${functionCatalog.name()}'.") + } + Identifier.of(Array(parts(1)), parts(2)) + case _ => + throw new UnsupportedOperationException( + "sourceTable must be database.table or catalog.database.table") + } + + val table = functionCatalog.asTableCatalog.loadTable(identifier) + if (!table.isInstanceOf[SparkTable]) { + throw new UnsupportedOperationException(s"$sourceTable is not a Paimon table") + } + val dataTable = table.asInstanceOf[SparkTable].table match { + case table: DataTable => table + case _ => + throw new UnsupportedOperationException(s"$sourceTable is not a Paimon data table") + } + val fileIO = dataTable.fileIO() + fileIO.isObjectStore + + ApplyFunctionExpression( + new ResolvedDescriptorToPresignedUrlFunction(fileIO, dataTable.location(), ignoreErrors), + Seq(descriptor, extension, validity)) + } + def resolveBlobView( spark: SparkSession, tableName: String, @@ -82,6 +128,8 @@ object ReplacePaimonFunctions { /** A rule to replace Paimon functions with literal values. */ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] { + private lazy val catalogManager = spark.sessionState.catalogManager + private def replaceMaxPt(func: ApplyFunctionExpression): Expression = { assert(func.children.size == 1) assert(func.children.head.dataType == StringType) @@ -138,6 +186,56 @@ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] ReplacePaimonFunctions.resolveBlobView(spark, tableName, fieldName, arguments(2)) } + private def replaceDescriptorToPresignedUrl( + arguments: Seq[Expression], + function: DescriptorToPresignedUrlFunction): Expression = { + assert(arguments.size == 4) + val sourceTable = literalString(arguments(0), "sourceTable") + if (sourceTable == null) { + throw new UnsupportedOperationException("sourceTable must not be null") + } + if (arguments(0).dataType != StringType) { + throw new UnsupportedOperationException("sourceTable must be STRING type") + } + + val descriptor = castNullable(arguments(1), BinaryType, "descriptor") + val extension = castNullable(arguments(2), StringType, "extension") + val intervalType = DayTimeIntervalType.DEFAULT + val validity = arguments(3).dataType match { + case _: DayTimeIntervalType => Cast(arguments(3), intervalType) + case NullType => Cast(arguments(3), intervalType) + case other => + throw new UnsupportedOperationException( + s"validity must be INTERVAL DAY TO SECOND type, but found ${other.simpleString}") + } + val functionCatalog = Option(function.catalogName()) + .map(catalogManager.catalog) + .getOrElse(catalogManager.currentCatalog) + + ReplacePaimonFunctions.resolveDescriptorToPresignedUrl( + spark, + functionCatalog, + sourceTable, + descriptor, + extension, + validity, + function.ignoreErrors()) + } + + private def castNullable( + expression: Expression, + expectedType: DataType, + argumentName: String): Expression = { + expression.dataType match { + case NullType => Cast(expression, expectedType) + case actual if actual == expectedType => expression + case actual => + throw new UnsupportedOperationException( + s"$argumentName must be ${expectedType.simpleString.toUpperCase} type, " + + s"but found ${actual.simpleString}") + } + } + private def literalString(child: Expression, argumentName: String): String = { if (!child.isInstanceOf[Literal]) { throw new UnsupportedOperationException(s"$argumentName must be a literal") @@ -158,6 +256,18 @@ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] } } + private def descriptorToPresignedUrlFunction( + invoke: Invoke): Option[DescriptorToPresignedUrlFunction] = { + if (invoke.functionName != "invoke" || !invoke.targetObject.foldable) { + None + } else { + invoke.targetObject.eval() match { + case function: DescriptorToPresignedUrlFunction => Some(function) + case _ => None + } + } + } + override def apply(plan: LogicalPlan): LogicalPlan = { AnalysisHelper.allowInvokingTransformsInAnalyzer { plan.transformAllExpressions { @@ -169,8 +279,17 @@ case class ReplacePaimonFunctions(spark: SparkSession) extends Rule[LogicalPlan] if func.function.name() == PaimonFunctions.BLOB_VIEW && func.function.canonicalName().startsWith("paimon") => replaceBlobView(func.children) + case func: ApplyFunctionExpression + if func.function.isInstanceOf[DescriptorToPresignedUrlFunction] => + replaceDescriptorToPresignedUrl( + func.children, + func.function.asInstanceOf[DescriptorToPresignedUrlFunction]) case invoke: Invoke if isBlobViewInvoke(invoke) => replaceBlobView(invoke.arguments) + case invoke: Invoke if descriptorToPresignedUrlFunction(invoke).isDefined => + replaceDescriptorToPresignedUrl( + invoke.arguments, + descriptorToPresignedUrlFunction(invoke).get) } } } diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunctionTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunctionTest.java new file mode 100644 index 000000000000..c05663c97760 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/function/DescriptorToPresignedUrlFunctionTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.function; + +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.utils.BlobDescriptorUtils; + +import org.apache.spark.sql.connector.catalog.functions.BoundFunction; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for descriptor-to-presigned-URL Spark functions. */ +class DescriptorToPresignedUrlFunctionTest { + + private static final Path TABLE_ROOT = new Path("oss://bucket/table"); + private static final BlobDescriptor DESCRIPTOR = + new BlobDescriptor("oss://bucket/table/bucket-0/data.blob", 10, 20); + + @Test + void testBindAndNonDeterministic() { + StructType inputType = + new StructType() + .add("source_table", DataTypes.StringType) + .add("descriptor", DataTypes.BinaryType) + .add("extension", DataTypes.StringType) + .add("validity", DataTypes.createDayTimeIntervalType()); + + BoundFunction strict = new DescriptorToPresignedUrlUnbound(false).bind(inputType); + BoundFunction lenient = new DescriptorToPresignedUrlUnbound(true).bind(inputType); + + assertThat(strict).isInstanceOf(DescriptorToPresignedUrlFunction.class); + assertThat(strict.isDeterministic()).isFalse(); + assertThat(lenient.isDeterministic()).isFalse(); + assertThat(strict.inputTypes()) + .containsExactly( + DataTypes.StringType, + DataTypes.BinaryType, + DataTypes.StringType, + DataTypes.createDayTimeIntervalType()); + assertThatThrownBy( + () -> + new DescriptorToPresignedUrlUnbound(false) + .bind( + new StructType() + .add("source_table", DataTypes.StringType) + .add("descriptor", DataTypes.BinaryType) + .add("extension", DataTypes.StringType))) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testResolvedFunction() throws IOException { + ResolvedDescriptorToPresignedUrlFunction function = + new ResolvedDescriptorToPresignedUrlFunction( + new TestingFileIO(), TABLE_ROOT, false); + + assertThat( + function.invoke( + DESCRIPTOR.serialize(), UTF8String.fromString("png"), 3_000_000L)) + .hasToString("https://example.test/data.png?validity=3&offset=10&length=20"); + assertThat(function.isDeterministic()).isFalse(); + } + + @Test + void testNullArguments() throws IOException { + ResolvedDescriptorToPresignedUrlFunction function = + new ResolvedDescriptorToPresignedUrlFunction( + new TestingFileIO(), TABLE_ROOT, false); + + assertThat(function.invoke(null, UTF8String.fromString("png"), 1_000_000L)).isNull(); + assertThat(function.invoke(DESCRIPTOR.serialize(), null, 1_000_000L)).isNull(); + } + + @Test + void testValidityMustBePositiveWholeSeconds() { + ResolvedDescriptorToPresignedUrlFunction function = + new ResolvedDescriptorToPresignedUrlFunction( + new TestingFileIO(), TABLE_ROOT, false); + + assertThatThrownBy( + () -> + function.invoke( + DESCRIPTOR.serialize(), + UTF8String.fromString("png"), + 1_000_001L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("whole seconds"); + assertThatThrownBy( + () -> + function.invoke( + DESCRIPTOR.serialize(), UTF8String.fromString("png"), 0L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("positive"); + } + + @Test + void testTryReturnsNullOnRowError() throws IOException { + ResolvedDescriptorToPresignedUrlFunction function = + new ResolvedDescriptorToPresignedUrlFunction(new TestingFileIO(), TABLE_ROOT, true); + + assertThat(function.invoke(new byte[] {1}, UTF8String.fromString("png"), 1_000_000L)) + .isNull(); + assertThat( + function.invoke( + DESCRIPTOR.serialize(), UTF8String.fromString("png"), 1_000_001L)) + .isNull(); + } + + @Test + void testRejectDescriptorOutsideTableRoot() { + ResolvedDescriptorToPresignedUrlFunction function = + new ResolvedDescriptorToPresignedUrlFunction( + new TestingFileIO(), TABLE_ROOT, false); + BlobDescriptor other = new BlobDescriptor("oss://bucket/other/bucket-0/data.blob", 0, 1); + + assertThatThrownBy( + () -> + function.invoke( + other.serialize(), + UTF8String.fromString("png"), + 1_000_000L)) + .isInstanceOf(IOException.class) + .hasMessageContaining("under table root"); + } + + @Test + void testFileIOSerialization() throws Exception { + ResolvedDescriptorToPresignedUrlFunction original = + new ResolvedDescriptorToPresignedUrlFunction( + new TestingFileIO(), TABLE_ROOT, false); + + ResolvedDescriptorToPresignedUrlFunction restored = roundTrip(original); + + assertThat( + restored.invoke( + DESCRIPTOR.serialize(), UTF8String.fromString("jpg"), 2_000_000L)) + .hasToString("https://example.test/data.jpg?validity=2&offset=10&length=20"); + } + + private ResolvedDescriptorToPresignedUrlFunction roundTrip( + ResolvedDescriptorToPresignedUrlFunction function) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(function); + } + try (ObjectInputStream input = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (ResolvedDescriptorToPresignedUrlFunction) input.readObject(); + } + } + + private static class TestingFileIO extends LocalFileIO { + + private static final long serialVersionUID = 1L; + + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + BlobDescriptorUtils.validateTableRoot(tableRoot, descriptor); + return "https://example.test/data." + + extension + + "?validity=" + + validity.getSeconds() + + "&offset=" + + descriptor.offset() + + "&length=" + + descriptor.length(); + } + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala index 765eb136bba4..7b667e4ed33f 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark.sql +import org.apache.paimon.data.BlobDescriptor import org.apache.paimon.spark.PaimonHiveTestBase import org.apache.paimon.spark.catalog.functions.PaimonFunctions import org.apache.paimon.spark.function.FunctionResources.MyIntSumClass @@ -114,6 +115,64 @@ class PaimonFunctionTest extends PaimonHiveTestBase { checkAnswer(sql(s"SHOW FUNCTIONS FROM $hiveDbName LIKE 'myIntSum'"), Seq.empty) } + test("Paimon function: descriptor to presigned URL") { + sql(s"USE paimon.$dbName0") + withTable("url_source", "url_other") { + sql("CREATE TABLE url_source (id INT) USING paimon") + sql("CREATE TABLE url_other (id INT) USING paimon") + + val descriptor = new BlobDescriptor("file:/tmp/source.blob", 0, 1).serialize() + val descriptorHex = descriptor.map(b => f"${b & 0xff}%02X").mkString + + checkAnswer( + sql( + s"SELECT sys.try_descriptor_to_presigned_url(" + + s"'$dbName0.url_source', CAST(NULL AS BINARY), 'png', INTERVAL '1' SECOND)"), + Row(null)) + checkAnswer( + sql( + s"SELECT sys.try_descriptor_to_presigned_url(" + + s"'paimon.$dbName0.url_source', X'00', 'png', INTERVAL '1' HOUR)"), + Row(null)) + + intercept[Exception] { + sql( + s"SELECT sys.descriptor_to_presigned_url(" + + s"'$dbName0.url_source', X'00', 'png', INTERVAL '1' SECOND)").collect() + } + intercept[Exception] { + sql( + s"SELECT sys.descriptor_to_presigned_url(" + + s"'$dbName0.url_source', X'$descriptorHex', 'png', " + + "INTERVAL '1.000001' SECOND)").collect() + } + intercept[Exception] { + sql( + s"SELECT sys.try_descriptor_to_presigned_url(" + + s"source_table, X'00', 'png', INTERVAL '1' SECOND) " + + s"FROM VALUES ('$dbName0.url_source') AS t(source_table)").collect() + } + intercept[Exception] { + sql( + s"SELECT sys.try_descriptor_to_presigned_url(" + + s"CASE WHEN id = 1 THEN '$dbName0.url_source' " + + s"ELSE '$dbName0.url_other' END, X'00', 'png', INTERVAL '1' SECOND) " + + "FROM VALUES (1) AS t(id)").collect() + } + intercept[Exception] { + sql( + "SELECT sys.try_descriptor_to_presigned_url(" + + "CAST(NULL AS STRING), X'00', 'png', INTERVAL '1' SECOND)").collect() + } + intercept[Exception] { + sql( + s"SELECT sys.try_descriptor_to_presigned_url(" + + s"'spark_catalog.$dbName0.url_source', X'00', 'png', " + + "INTERVAL '1' SECOND)").collect() + } + } + } + test("Add max_pt function") { Seq("paimon", sparkCatalogName, paimonHiveCatalogName).foreach { catalogName => From 26fa59c8bbd00ee9213574f52f578bf1c02c0182 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 23 Jul 2026 19:09:42 +0800 Subject: [PATCH 2/5] [flink] Add expression pushdown compatibility shim --- ...tsExpressionPushDownCompatibilityTest.java | 42 +++++++++++++++++++ .../abilities/SupportsExpressionPushDown.java | 31 ++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SupportsExpressionPushDownCompatibilityTest.java create mode 100644 paimon-flink/paimon-flink1-common/src/main/java/org/apache/flink/table/connector/source/abilities/SupportsExpressionPushDown.java diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SupportsExpressionPushDownCompatibilityTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SupportsExpressionPushDownCompatibilityTest.java new file mode 100644 index 000000000000..dd49027413dc --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SupportsExpressionPushDownCompatibilityTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.flink.runtime.util.EnvironmentInformation; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +class SupportsExpressionPushDownCompatibilityTest { + + @Test + void testCompatibilityClassIsAvailable() throws Exception { + assumeTrue(EnvironmentInformation.getVersion().startsWith("1.")); + + Class ability = + Class.forName( + "org.apache.flink.table.connector.source.abilities.SupportsExpressionPushDown"); + + assertThat(ability.getMethod("applyExpressions", List.class).getReturnType()) + .isEqualTo(List.class); + } +} diff --git a/paimon-flink/paimon-flink1-common/src/main/java/org/apache/flink/table/connector/source/abilities/SupportsExpressionPushDown.java b/paimon-flink/paimon-flink1-common/src/main/java/org/apache/flink/table/connector/source/abilities/SupportsExpressionPushDown.java new file mode 100644 index 000000000000..50b74f2ea4f5 --- /dev/null +++ b/paimon-flink/paimon-flink1-common/src/main/java/org/apache/flink/table/connector/source/abilities/SupportsExpressionPushDown.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.connector.source.abilities; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.expressions.ResolvedExpression; + +import java.util.List; + +/** Enables a table source to push down resolved expressions. */ +@PublicEvolving +public interface SupportsExpressionPushDown { + + List applyExpressions(List expressions); +} From c93a771cb04ce0e8a76fdc731fbb25ed494aa7a4 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 23 Jul 2026 19:22:39 +0800 Subject: [PATCH 3/5] [jindo] Support blob presigned URLs --- paimon-filesystems/paimon-jindo/pom.xml | 8 +- .../org/apache/paimon/jindo/JindoFileIO.java | 20 +++++ .../apache/paimon/jindo/JindoFileIOTest.java | 81 +++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java diff --git a/paimon-filesystems/paimon-jindo/pom.xml b/paimon-filesystems/paimon-jindo/pom.xml index d9d008cc756b..a419836c5cb4 100644 --- a/paimon-filesystems/paimon-jindo/pom.xml +++ b/paimon-filesystems/paimon-jindo/pom.xml @@ -50,6 +50,12 @@ provided + + org.apache.paimon + paimon-oss + ${project.version} + + org.apache.hadoop hadoop-common @@ -124,4 +130,4 @@ test - \ No newline at end of file + diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 7cab897e7654..963905572e72 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -19,11 +19,13 @@ package org.apache.paimon.jindo; import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.HadoopOptionsProvider; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.options.Options; +import org.apache.paimon.oss.OSSLoader; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SensitiveConfigUtils; @@ -41,6 +43,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -90,6 +93,15 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private Options hadoopOptions; private Options hadoopOptionsWithCache; private boolean allowCache = true; + private final FileIO blobFileIO; + + public JindoFileIO() { + this(new OSSLoader().load(new Path("oss:///"))); + } + + JindoFileIO(FileIO blobFileIO) { + this.blobFileIO = blobFileIO; + } @Override public boolean isObjectStore() { @@ -99,6 +111,7 @@ public boolean isObjectStore() { @Override public void configure(CatalogContext context) { super.configure(context); + blobFileIO.configure(context); allowCache = context.options().get(FILE_IO_ALLOW_CACHE); hadoopOptions = new Options(); // read all configuration with prefix 'CONFIG_PREFIXES' @@ -198,6 +211,13 @@ public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite new JindoMultiPartUpload(fs, hadoopPath), hadoopPath, path); } + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + return blobFileIO.createBlobPresignedUrl(tableRoot, descriptor, extension, validity); + } + @Override protected Pair createFileSystem( org.apache.hadoop.fs.Path path, boolean enableCache) { diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java new file mode 100644 index 000000000000..85836ebd6f07 --- /dev/null +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.jindo; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link JindoFileIO}. */ +public class JindoFileIOTest { + + @Test + public void testCreateBlobPresignedUrlDelegates() throws Exception { + TestingFileIO delegate = new TestingFileIO(); + JindoFileIO fileIO = new JindoFileIO(delegate); + CatalogContext context = CatalogContext.create(new Options()); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/data/file", 10, 20); + Duration validity = Duration.ofHours(1); + + fileIO.configure(context); + + assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "jpg", validity)) + .isEqualTo("https://bucket.oss.example/blob.jpg"); + assertThat(delegate.context).isSameAs(context); + assertThat(delegate.tableRoot).isEqualTo(tableRoot); + assertThat(delegate.descriptor).isEqualTo(descriptor); + assertThat(delegate.extension).isEqualTo("jpg"); + assertThat(delegate.validity).isEqualTo(validity); + } + + private static class TestingFileIO extends LocalFileIO { + + private CatalogContext context; + private Path tableRoot; + private BlobDescriptor descriptor; + private String extension; + private Duration validity; + + @Override + public void configure(CatalogContext context) { + this.context = context; + } + + @Override + public String createBlobPresignedUrl( + Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) + throws IOException { + this.tableRoot = tableRoot; + this.descriptor = descriptor; + this.extension = extension; + this.validity = validity; + return "https://bucket.oss.example/blob.jpg"; + } + } +} From 52c135d677492e85671c43bdd2a257c3dc10f5f6 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 23 Jul 2026 21:17:43 +0800 Subject: [PATCH 4/5] [jindo] Use configured OSS client for blob URLs --- paimon-filesystems/paimon-jindo/pom.xml | 60 +++- .../org/apache/paimon/jindo/JindoFileIO.java | 47 ++- .../apache/paimon/jindo/JindoFileIOTest.java | 98 ++++--- .../apache/paimon/oss/OSSBlobPresigner.java | 267 ++++++++++++++++++ .../java/org/apache/paimon/oss/OSSFileIO.java | 217 +------------- 5 files changed, 428 insertions(+), 261 deletions(-) create mode 100644 paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java diff --git a/paimon-filesystems/paimon-jindo/pom.xml b/paimon-filesystems/paimon-jindo/pom.xml index a419836c5cb4..bfb3faac6a90 100644 --- a/paimon-filesystems/paimon-jindo/pom.xml +++ b/paimon-filesystems/paimon-jindo/pom.xml @@ -32,6 +32,7 @@ jar + 3.13.2 6.9.1 @@ -52,10 +53,29 @@ org.apache.paimon - paimon-oss + paimon-oss-impl ${project.version} + + com.aliyun.oss + aliyun-sdk-oss + ${fs.oss.sdk.version} + + + org.jacoco + org.jacoco.agent + + + + + + org.mockito + mockito-core + ${mockito.version} + test + + org.apache.hadoop hadoop-common @@ -127,7 +147,43 @@ commons-logging commons-logging 1.1.3 - test + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + org.apache.paimon:paimon-oss-impl + com.aliyun.oss:aliyun-sdk-oss + com.aliyun:* + com.google.code.gson:gson + commons-codec:commons-codec + commons-logging:commons-logging + io.opentracing:* + javax.activation:javax.activation-api + javax.xml.bind:jaxb-api + org.apache.httpcomponents:* + org.codehaus.jettison:jettison + org.ini4j:ini4j + org.jdom:jdom2 + stax:stax-api + + + + + org.apache.paimon:paimon-oss-impl + + org/apache/paimon/oss/OSSBlobPresigner.class + + + + + + + diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java index 963905572e72..426605b0681a 100644 --- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java +++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java @@ -25,7 +25,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.options.Options; -import org.apache.paimon.oss.OSSLoader; +import org.apache.paimon.oss.OSSBlobPresigner; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SensitiveConfigUtils; @@ -35,6 +35,8 @@ import com.aliyun.jindodata.dls.JindoDlsFileSystem; import com.aliyun.jindodata.oss.JindoOssFileSystem; import com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSClientBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.slf4j.Logger; @@ -67,6 +69,7 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP */ private static final String[] CONFIG_PREFIXES = {"fs."}; + private static final String OSS_ENDPOINT = "fs.oss.endpoint"; private static final String OSS_ACCESS_KEY_ID = "fs.oss.accessKeyId"; private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; @@ -93,14 +96,12 @@ public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsP private Options hadoopOptions; private Options hadoopOptionsWithCache; private boolean allowCache = true; - private final FileIO blobFileIO; + private transient OSSClient blobClient; - public JindoFileIO() { - this(new OSSLoader().load(new Path("oss:///"))); - } + public JindoFileIO() {} - JindoFileIO(FileIO blobFileIO) { - this.blobFileIO = blobFileIO; + JindoFileIO(OSSClient blobClient) { + this.blobClient = blobClient; } @Override @@ -111,7 +112,6 @@ public boolean isObjectStore() { @Override public void configure(CatalogContext context) { super.configure(context); - blobFileIO.configure(context); allowCache = context.options().get(FILE_IO_ALLOW_CACHE); hadoopOptions = new Options(); // read all configuration with prefix 'CONFIG_PREFIXES' @@ -215,7 +215,30 @@ public TwoPhaseOutputStream newTwoPhaseOutputStream(Path path, boolean overwrite public String createBlobPresignedUrl( Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) throws IOException { - return blobFileIO.createBlobPresignedUrl(tableRoot, descriptor, extension, validity); + return OSSBlobPresigner.create(blobClient(), tableRoot, descriptor, extension, validity); + } + + private synchronized OSSClient blobClient() { + if (blobClient == null) { + blobClient = createBlobClient(hadoopOptions); + } + return blobClient; + } + + static OSSClient createBlobClient(Options options) { + String securityToken = options.get(OSS_SECURITY_TOKEN); + OSSClientBuilder builder = new OSSClientBuilder(); + return (OSSClient) + (StringUtils.isNullOrWhitespaceOnly(securityToken) + ? builder.build( + options.get(OSS_ENDPOINT), + options.get(OSS_ACCESS_KEY_ID), + options.get(OSS_ACCESS_KEY_SECRET)) + : builder.build( + options.get(OSS_ENDPOINT), + options.get(OSS_ACCESS_KEY_ID), + options.get(OSS_ACCESS_KEY_SECRET), + securityToken)); } @Override @@ -266,7 +289,11 @@ protected Pair createFileSystem( } @Override - public void close() { + public synchronized void close() { + if (blobClient != null) { + blobClient.shutdown(); + blobClient = null; + } if (!allowCache) { fsMap.values().stream().map(Pair::getKey).forEach(IOUtils::closeQuietly); fsMap.clear(); diff --git a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java index 85836ebd6f07..b5cb1aad014d 100644 --- a/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java +++ b/paimon-filesystems/paimon-jindo/src/test/java/org/apache/paimon/jindo/JindoFileIOTest.java @@ -18,64 +18,86 @@ package org.apache.paimon.jindo; -import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.fs.Path; -import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import com.aliyun.oss.HttpMethod; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.model.ObjectMetadata; import org.junit.jupiter.api.Test; -import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.security.MessageDigest; import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Tests for {@link JindoFileIO}. */ public class JindoFileIOTest { @Test - public void testCreateBlobPresignedUrlDelegates() throws Exception { - TestingFileIO delegate = new TestingFileIO(); - JindoFileIO fileIO = new JindoFileIO(delegate); - CatalogContext context = CatalogContext.create(new Options()); - Path tableRoot = new Path("oss://bucket/table"); - BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/data/file", 10, 20); - Duration validity = Duration.ofHours(1); - - fileIO.configure(context); + public void testCreateBlobClientUsesConfiguredSts() { + Options options = new Options(); + options.set("fs.oss.endpoint", "https://oss.example.com"); + options.set("fs.oss.accessKeyId", "access-key"); + options.set("fs.oss.accessKeySecret", "access-secret"); + options.set("fs.oss.securityToken", "security-token"); - assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "jpg", validity)) - .isEqualTo("https://bucket.oss.example/blob.jpg"); - assertThat(delegate.context).isSameAs(context); - assertThat(delegate.tableRoot).isEqualTo(tableRoot); - assertThat(delegate.descriptor).isEqualTo(descriptor); - assertThat(delegate.extension).isEqualTo("jpg"); - assertThat(delegate.validity).isEqualTo(validity); + OSSClient client = JindoFileIO.createBlobClient(options); + try { + assertThat(client.getEndpoint()).isEqualTo(URI.create("https://oss.example.com")); + assertThat(client.getCredentialsProvider().getCredentials().getAccessKeyId()) + .isEqualTo("access-key"); + assertThat(client.getCredentialsProvider().getCredentials().getSecretAccessKey()) + .isEqualTo("access-secret"); + assertThat(client.getCredentialsProvider().getCredentials().getSecurityToken()) + .isEqualTo("security-token"); + } finally { + client.shutdown(); + } } - private static class TestingFileIO extends LocalFileIO { + @Test + public void testCreateBlobPresignedUrlUsesOssClient() throws Exception { + OSSClient client = mock(OSSClient.class); + Path tableRoot = new Path("oss://bucket/table"); + BlobDescriptor descriptor = new BlobDescriptor("oss://bucket/table/data/file", 10, 20); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(20); + metadata.setContentType("image/jpeg"); + metadata.addUserMetadata( + "paimon-blob-descriptor-sha256", sha256Hex(descriptor.serialize())); + when(client.headObject(eq("bucket"), contains("_bloburl_"))).thenReturn(metadata); + when(client.getEndpoint()).thenReturn(URI.create("https://oss.example.com")); + when(client.generatePresignedUrl(eq("bucket"), anyString(), any(), eq(HttpMethod.GET))) + .thenAnswer( + invocation -> + new URL( + "https://bucket.oss.example.com/" + + invocation.getArgument(1))); - private CatalogContext context; - private Path tableRoot; - private BlobDescriptor descriptor; - private String extension; - private Duration validity; + JindoFileIO fileIO = new JindoFileIO(client); + assertThat(fileIO.createBlobPresignedUrl(tableRoot, descriptor, "jpg", Duration.ofHours(1))) + .startsWith("https://bucket.oss.example.com/table/data/_bloburl_"); - @Override - public void configure(CatalogContext context) { - this.context = context; - } + fileIO.close(); + verify(client).shutdown(); + } - @Override - public String createBlobPresignedUrl( - Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) - throws IOException { - this.tableRoot = tableRoot; - this.descriptor = descriptor; - this.extension = extension; - this.validity = validity; - return "https://bucket.oss.example/blob.jpg"; + private static String sha256Hex(byte[] bytes) throws Exception { + StringBuilder result = new StringBuilder(); + for (byte value : MessageDigest.getInstance("SHA-256").digest(bytes)) { + result.append(String.format("%02x", value & 0xff)); } + return result.toString(); } } diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java new file mode 100644 index 000000000000..5d3914abd973 --- /dev/null +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSBlobPresigner.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.oss; + +import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.fs.Path; +import org.apache.paimon.utils.BlobDescriptorUtils; + +import com.aliyun.oss.HttpMethod; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PartETag; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.net.URLConnection; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +/** Materializes blob ranges and creates OSS presigned URLs. */ +public final class OSSBlobPresigner { + + private static final String BLOB_FINGERPRINT_METADATA = "paimon-blob-descriptor-sha256"; + private static final long BLOB_COPY_MIN_PART_SIZE = 100L * 1024 * 1024; + private static final long MAX_MULTIPART_UPLOAD_PARTS = 10_000; + private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); + + private OSSBlobPresigner() {} + + public static String create( + OSSClient client, + Path tableRoot, + BlobDescriptor descriptor, + String extension, + Duration validity) + throws IOException { + BlobDescriptorUtils.validateTableRoot(tableRoot, descriptor); + if (validity == null + || validity.isZero() + || validity.isNegative() + || validity.getNano() != 0) { + throw new IOException("Blob presigned URL validity must be positive whole seconds."); + } + + String normalizedExtension = normalizeExtension(extension); + String contentType = contentType(normalizedExtension); + try { + URI source = new Path(descriptor.uri()).toUri(); + String bucket = source.getAuthority(); + String sourceKey = objectKey(source); + String fingerprint = sha256Hex(descriptor.serialize()); + int parentEnd = sourceKey.lastIndexOf('/') + 1; + String targetKey = + sourceKey.substring(0, parentEnd) + + "_bloburl_" + + fingerprint + + "." + + normalizedExtension; + + ObjectMetadata target = headObjectIfExists(client, bucket, targetKey); + if (!matches(target, descriptor.length(), contentType, fingerprint)) { + ObjectMetadata sourceMetadata = client.headObject(bucket, sourceKey); + validateRange(descriptor, sourceMetadata.getContentLength()); + materialize( + client, bucket, sourceKey, targetKey, descriptor, contentType, fingerprint); + target = client.headObject(bucket, targetKey); + if (!matches(target, descriptor.length(), contentType, fingerprint)) { + throw new IOException( + "Materialized blob object metadata does not match descriptor."); + } + } + + URL url = + client.generatePresignedUrl( + bucket, + targetKey, + Date.from(Instant.now().plus(validity)), + HttpMethod.GET); + validatePresignedUrl(client, url, bucket, targetKey); + return url.toString(); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Failed to create blob presigned URL.", e); + } + } + + private static void materialize( + OSSClient client, + String bucket, + String sourceKey, + String targetKey, + BlobDescriptor descriptor, + String contentType, + String fingerprint) + throws Exception { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentType(contentType); + metadata.addUserMetadata(BLOB_FINGERPRINT_METADATA, fingerprint); + if (descriptor.length() == 0) { + metadata.setContentLength(0); + client.putObject(bucket, targetKey, new ByteArrayInputStream(new byte[0]), metadata); + return; + } + + String uploadId = null; + try { + InitiateMultipartUploadResult initiated = + client.initiateMultipartUpload( + new InitiateMultipartUploadRequest(bucket, targetKey, metadata)); + uploadId = initiated.getUploadId(); + long partSize = + Math.max( + BLOB_COPY_MIN_PART_SIZE, + descriptor.length() / MAX_MULTIPART_UPLOAD_PARTS + 1); + List parts = new ArrayList<>(); + long copied = 0; + int partNumber = 1; + while (copied < descriptor.length()) { + long size = Math.min(partSize, descriptor.length() - copied); + UploadPartCopyResult result = + client.uploadPartCopy( + new UploadPartCopyRequest( + bucket, + sourceKey, + bucket, + targetKey, + uploadId, + partNumber, + descriptor.offset() + copied, + size)); + parts.add(new PartETag(partNumber, result.getETag())); + copied += size; + partNumber++; + } + client.completeMultipartUpload( + new CompleteMultipartUploadRequest(bucket, targetKey, uploadId, parts)); + uploadId = null; + } catch (Exception e) { + if (uploadId != null) { + try { + client.abortMultipartUpload( + new AbortMultipartUploadRequest(bucket, targetKey, uploadId)); + } catch (Exception abortException) { + e.addSuppressed(abortException); + } + } + throw e; + } + } + + private static ObjectMetadata headObjectIfExists(OSSClient client, String bucket, String key) { + try { + return client.headObject(bucket, key); + } catch (OSSException e) { + if ("NoSuchKey".equals(e.getErrorCode()) || "NoSuchObject".equals(e.getErrorCode())) { + return null; + } + throw e; + } + } + + private static boolean matches( + ObjectMetadata metadata, long length, String contentType, String fingerprint) { + return metadata != null + && metadata.getContentLength() == length + && contentType.equals(metadata.getContentType()) + && fingerprint.equals(metadata.getUserMetadata().get(BLOB_FINGERPRINT_METADATA)); + } + + private static void validateRange(BlobDescriptor descriptor, long sourceLength) + throws IOException { + if (descriptor.offset() < 0 + || descriptor.length() < 0 + || descriptor.offset() > sourceLength + || descriptor.length() > sourceLength - descriptor.offset()) { + throw new IOException("Blob descriptor range is outside the source object."); + } + } + + private static String normalizeExtension(String extension) throws IOException { + if (extension == null || extension.isEmpty()) { + throw new IOException("Blob file extension must contain only ASCII letters or digits."); + } + for (int i = 0; i < extension.length(); i++) { + char c = extension.charAt(i); + if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9')) { + throw new IOException( + "Blob file extension must contain only ASCII letters or digits."); + } + } + return extension.toLowerCase(Locale.ROOT); + } + + private static String contentType(String extension) throws IOException { + String contentType = URLConnection.guessContentTypeFromName("file." + extension); + if (contentType == null || "application/octet-stream".equals(contentType)) { + throw new IOException("Unknown MIME type for blob file extension: " + extension); + } + return contentType; + } + + private static String objectKey(URI uri) throws IOException { + String path = uri.getPath(); + if (path == null || !path.startsWith("/") || path.length() == 1) { + throw new IOException("Blob descriptor URI must contain an OSS object key."); + } + return path.substring(1); + } + + private static String sha256Hex(byte[] bytes) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(bytes); + char[] chars = new char[hash.length * 2]; + for (int i = 0; i < hash.length; i++) { + chars[i * 2] = HEX_CHARS[(hash[i] >> 4) & 0x0f]; + chars[i * 2 + 1] = HEX_CHARS[hash[i] & 0x0f]; + } + return new String(chars); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 not available.", e); + } + } + + private static void validatePresignedUrl( + OSSClient client, URL url, String bucket, String targetKey) throws Exception { + URI endpoint = client.getEndpoint(); + String expectedHost = bucket + "." + endpoint.getHost(); + if (!"https".equalsIgnoreCase(endpoint.getScheme()) + || !"https".equalsIgnoreCase(url.getProtocol()) + || !expectedHost.equalsIgnoreCase(url.getHost()) + || !("/" + targetKey).equals(url.toURI().getPath())) { + throw new IOException("OSS client generated a presigned URL for an invalid target."); + } + } +} diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java index a0d02e3f7a4f..66ad3c9576a2 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java @@ -25,13 +25,11 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.TwoPhaseOutputStream; import org.apache.paimon.options.Options; -import org.apache.paimon.utils.BlobDescriptorUtils; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.ReflectionUtils; import org.apache.paimon.utils.SensitiveConfigUtils; import org.apache.paimon.utils.StringUtils; -import com.aliyun.oss.HttpMethod; import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSException; import com.aliyun.oss.common.auth.CredentialsProvider; @@ -39,23 +37,17 @@ import com.aliyun.oss.internal.OSSHeaders; import com.aliyun.oss.internal.OSSMultipartOperation; import com.aliyun.oss.internal.OSSObjectOperation; -import com.aliyun.oss.model.AbortMultipartUploadRequest; -import com.aliyun.oss.model.CompleteMultipartUploadRequest; import com.aliyun.oss.model.CopyObjectRequest; import com.aliyun.oss.model.CopyObjectResult; import com.aliyun.oss.model.InitiateMultipartUploadRequest; import com.aliyun.oss.model.InitiateMultipartUploadResult; import com.aliyun.oss.model.ObjectMetadata; -import com.aliyun.oss.model.PartETag; import com.aliyun.oss.model.PutObjectRequest; import com.aliyun.oss.model.PutObjectResult; -import com.aliyun.oss.model.UploadPartCopyRequest; -import com.aliyun.oss.model.UploadPartCopyResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore; -import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,18 +55,9 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; -import java.net.URL; -import java.net.URLConnection; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; -import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -100,9 +83,6 @@ public class OSSFileIO extends HadoopCompliantFileIO implements HadoopOptionsPro private static final String OSS_ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; private static final String OSS_SECOND_LEVEL_DOMAIN_ENABLED = "fs.oss.sld.enabled"; - private static final String BLOB_FINGERPRINT_METADATA = "paimon-blob-descriptor-sha256"; - private static final long BLOB_COPY_MIN_PART_SIZE = 100L * 1024 * 1024; - private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); // Paimon OSS SSE keys, mapping 1:1 to the OSS headers; they take precedence over hadoop's // server-side-encryption-algorithm. /** SSE method -> x-oss-server-side-encryption (AES256 / KMS / SM4). */ @@ -281,51 +261,13 @@ public boolean tryToWriteAtomic(Path path, String content) throws IOException { public String createBlobPresignedUrl( Path tableRoot, BlobDescriptor descriptor, String extension, Duration validity) throws IOException { - BlobDescriptorUtils.validateTableRoot(tableRoot, descriptor); - if (validity == null - || validity.isZero() - || validity.isNegative() - || validity.getNano() != 0) { - throw new IOException("Blob presigned URL validity must be positive whole seconds."); - } - - String normalizedExtension = normalizeExtension(extension); - String contentType = contentType(normalizedExtension); try { - URI source = new Path(descriptor.uri()).toUri(); - String bucket = source.getAuthority(); - String sourceKey = objectKey(source); - String fingerprint = sha256Hex(descriptor.serialize()); - int parentEnd = sourceKey.lastIndexOf('/') + 1; - String targetKey = - sourceKey.substring(0, parentEnd) - + "_bloburl_" - + fingerprint - + "." - + normalizedExtension; - OSSClient client = ossClient(new Path(descriptor.uri())); - - ObjectMetadata target = headObjectIfExists(client, bucket, targetKey); - if (!matches(target, descriptor.length(), contentType, fingerprint)) { - ObjectMetadata sourceMetadata = client.headObject(bucket, sourceKey); - validateRange(descriptor, sourceMetadata.getContentLength()); - materialize( - client, bucket, sourceKey, targetKey, descriptor, contentType, fingerprint); - target = client.headObject(bucket, targetKey); - if (!matches(target, descriptor.length(), contentType, fingerprint)) { - throw new IOException( - "Materialized blob object metadata does not match descriptor."); - } - } - - URL url = - client.generatePresignedUrl( - bucket, - targetKey, - Date.from(Instant.now().plus(validity)), - HttpMethod.GET); - validatePresignedUrl(client, url, bucket, targetKey); - return url.toString(); + return OSSBlobPresigner.create( + ossClient(new Path(descriptor.uri())), + tableRoot, + descriptor, + extension, + validity); } catch (IOException e) { throw e; } catch (Exception e) { @@ -333,153 +275,6 @@ public String createBlobPresignedUrl( } } - private static void materialize( - OSSClient client, - String bucket, - String sourceKey, - String targetKey, - BlobDescriptor descriptor, - String contentType, - String fingerprint) - throws Exception { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentType(contentType); - metadata.addUserMetadata(BLOB_FINGERPRINT_METADATA, fingerprint); - if (descriptor.length() == 0) { - metadata.setContentLength(0); - client.putObject(bucket, targetKey, new ByteArrayInputStream(new byte[0]), metadata); - return; - } - - String uploadId = null; - try { - InitiateMultipartUploadResult initiated = - client.initiateMultipartUpload( - new InitiateMultipartUploadRequest(bucket, targetKey, metadata)); - uploadId = initiated.getUploadId(); - long partSize = - AliyunOSSUtils.calculatePartSize(descriptor.length(), BLOB_COPY_MIN_PART_SIZE); - List parts = new ArrayList<>(); - long copied = 0; - int partNumber = 1; - while (copied < descriptor.length()) { - long size = Math.min(partSize, descriptor.length() - copied); - UploadPartCopyResult result = - client.uploadPartCopy( - new UploadPartCopyRequest( - bucket, - sourceKey, - bucket, - targetKey, - uploadId, - partNumber, - descriptor.offset() + copied, - size)); - parts.add(new PartETag(partNumber, result.getETag())); - copied += size; - partNumber++; - } - client.completeMultipartUpload( - new CompleteMultipartUploadRequest(bucket, targetKey, uploadId, parts)); - uploadId = null; - } catch (Exception e) { - if (uploadId != null) { - try { - client.abortMultipartUpload( - new AbortMultipartUploadRequest(bucket, targetKey, uploadId)); - } catch (Exception abortException) { - e.addSuppressed(abortException); - } - } - throw e; - } - } - - private static ObjectMetadata headObjectIfExists(OSSClient client, String bucket, String key) { - try { - return client.headObject(bucket, key); - } catch (OSSException e) { - if ("NoSuchKey".equals(e.getErrorCode()) || "NoSuchObject".equals(e.getErrorCode())) { - return null; - } - throw e; - } - } - - private static boolean matches( - ObjectMetadata metadata, long length, String contentType, String fingerprint) { - return metadata != null - && metadata.getContentLength() == length - && contentType.equals(metadata.getContentType()) - && fingerprint.equals(metadata.getUserMetadata().get(BLOB_FINGERPRINT_METADATA)); - } - - private static void validateRange(BlobDescriptor descriptor, long sourceLength) - throws IOException { - if (descriptor.offset() < 0 - || descriptor.length() < 0 - || descriptor.offset() > sourceLength - || descriptor.length() > sourceLength - descriptor.offset()) { - throw new IOException("Blob descriptor range is outside the source object."); - } - } - - private static String normalizeExtension(String extension) throws IOException { - if (extension == null || extension.isEmpty()) { - throw new IOException("Blob file extension must contain only ASCII letters or digits."); - } - for (int i = 0; i < extension.length(); i++) { - char c = extension.charAt(i); - if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9')) { - throw new IOException( - "Blob file extension must contain only ASCII letters or digits."); - } - } - return extension.toLowerCase(Locale.ROOT); - } - - private static String contentType(String extension) throws IOException { - String contentType = URLConnection.guessContentTypeFromName("file." + extension); - if (contentType == null || "application/octet-stream".equals(contentType)) { - throw new IOException("Unknown MIME type for blob file extension: " + extension); - } - return contentType; - } - - private static String objectKey(URI uri) throws IOException { - String path = uri.getPath(); - if (path == null || !path.startsWith("/") || path.length() == 1) { - throw new IOException("Blob descriptor URI must contain an OSS object key."); - } - return path.substring(1); - } - - private static String sha256Hex(byte[] bytes) { - try { - byte[] hash = MessageDigest.getInstance("SHA-256").digest(bytes); - char[] chars = new char[hash.length * 2]; - for (int i = 0; i < hash.length; i++) { - chars[i * 2] = HEX_CHARS[(hash[i] >> 4) & 0x0f]; - chars[i * 2 + 1] = HEX_CHARS[hash[i] & 0x0f]; - } - return new String(chars); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("SHA-256 not available.", e); - } - } - - private static void validatePresignedUrl( - OSSClient client, URL url, String bucket, String targetKey) throws Exception { - URI endpoint = client.getEndpoint(); - String expectedHost = bucket + "." + endpoint.getHost(); - if (!"https".equalsIgnoreCase(endpoint.getScheme()) - || !"https".equalsIgnoreCase(url.getProtocol()) - || !expectedHost.equalsIgnoreCase(url.getHost()) - || !("/" + targetKey).equals(url.toURI().getPath())) { - throw new IOException("OSS client generated a presigned URL for an invalid target."); - } - } - OSSClient ossClient(Path path) throws Exception { return getOssClient((AliyunOSSFileSystem) getFileSystem(path(path))); } From 28d06f9bb27eb70036e198f5bf4c985d474fa507 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 24 Jul 2026 11:22:13 +0800 Subject: [PATCH 5/5] fix --- paimon-filesystems/paimon-jindo/pom.xml | 19 ++++++- .../tools/ci/licensecheck/JarFileChecker.java | 10 ++++ .../ci/licensecheck/JarFileCheckerTest.java | 50 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/paimon-filesystems/paimon-jindo/pom.xml b/paimon-filesystems/paimon-jindo/pom.xml index bfb3faac6a90..5a6020a9d192 100644 --- a/paimon-filesystems/paimon-jindo/pom.xml +++ b/paimon-filesystems/paimon-jindo/pom.xml @@ -66,6 +66,10 @@ org.jacoco org.jacoco.agent + + javax.xml.bind + jaxb-api + @@ -152,6 +156,18 @@ + + org.apache.maven.plugins + maven-jar-plugin + + + + true + + + + + org.apache.maven.plugins maven-shade-plugin @@ -165,8 +181,6 @@ commons-codec:commons-codec commons-logging:commons-logging io.opentracing:* - javax.activation:javax.activation-api - javax.xml.bind:jaxb-api org.apache.httpcomponents:* org.codehaus.jettison:jettison org.ini4j:ini4j @@ -179,6 +193,7 @@ org.apache.paimon:paimon-oss-impl org/apache/paimon/oss/OSSBlobPresigner.class + META-INF/versions/11/** diff --git a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java index 386f2407b3c2..ac5785ec101a 100644 --- a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java +++ b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java @@ -201,10 +201,20 @@ private static int findNonBinaryFilesContainingText( .filter(path -> !getFileName(path).startsWith("notice")) // dual-licensed under GPL 2 and CDDL 1.1 // contained in hadoop/presto S3 FS and paimon-dist + .filter(path -> !pathStartsWith(path, "/javax/xml/bind")) .filter( path -> !path.toString() .contains("/META-INF/versions/11/javax/xml/bind")) + .filter( + path -> + !pathStartsWith( + path, + "/META-INF/maven/javax.activation/javax.activation-api")) + .filter( + path -> + !pathStartsWith( + path, "/META-INF/maven/javax.xml.bind/jaxb-api")) .filter(path -> !isJavaxManifest(jar, path)) // dual-licensed under GPL 2 and EPL 2.0 // contained in sql-avro-confluent-registry diff --git a/tools/ci/paimon-ci-tools/src/test/java/org/apache/paimon/tools/ci/licensecheck/JarFileCheckerTest.java b/tools/ci/paimon-ci-tools/src/test/java/org/apache/paimon/tools/ci/licensecheck/JarFileCheckerTest.java index 16fa96109e08..2b3165b0c853 100644 --- a/tools/ci/paimon-ci-tools/src/test/java/org/apache/paimon/tools/ci/licensecheck/JarFileCheckerTest.java +++ b/tools/ci/paimon-ci-tools/src/test/java/org/apache/paimon/tools/ci/licensecheck/JarFileCheckerTest.java @@ -248,6 +248,56 @@ void testForbiddenLGPMultiLineLongTextWithCommentAndLeadingWhitespaceDetected( .isEqualTo(1); } + @Test + void testIgnoreDualLicensedJavaxFiles(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "GNU General Public License", + Arrays.asList( + "javax", + "xml", + "bind", + "Messages.properties")), + Entry.fileEntry( + "GNU General Public License", + Arrays.asList( + "javax", + "xml", + "bind", + "helpers", + "Messages.properties")), + Entry.fileEntry( + "GNU General Public License", + Arrays.asList( + "javax", + "xml", + "bind", + "util", + "Messages.properties")), + Entry.fileEntry( + "GNU General Public License", + Arrays.asList( + "META-INF", + "maven", + "javax.activation", + "javax.activation-api", + "pom.xml")), + Entry.fileEntry( + "GNU General Public License", + Arrays.asList( + "META-INF", + "maven", + "javax.xml.bind", + "jaxb-api", + "pom.xml"))))) + .isEqualTo(0); + } + private static class Entry { final String contents; final List path;