Add presigned URL download support to S3 Transfer Manager

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java

@@ -55,11 +55,13 @@ public class S3IntegrationTestBase extends AwsTestBase {
     protected static S3Client s3;
 
     protected static S3AsyncClient s3Async;
+    protected static S3AsyncClient s3NonMultipartAsync;
 
     protected static S3AsyncClient s3CrtAsync;
 
     protected static S3TransferManager tmCrt;
     protected static S3TransferManager tmJava;
+    protected static S3TransferManager tmNonMultipartJava;
 
     /**
      * Loads the AWS account info for the integration tests and creates an S3
@@ -71,6 +73,11 @@ public static void setUpForAllIntegTests() throws Exception {
         System.setProperty("aws.crt.debugnative", "true");
         s3 = s3ClientBuilder().build();
         s3Async = s3AsyncClientBuilder().build();
+        s3NonMultipartAsync = S3AsyncClient.builder()
+                                           .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
+                                           .region(DEFAULT_REGION)
+                                           .build();
+
         s3CrtAsync = S3CrtAsyncClient.builder()
                                      .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
                                      .region(DEFAULT_REGION)
@@ -82,15 +89,21 @@ public static void setUpForAllIntegTests() throws Exception {
         tmJava = S3TransferManager.builder()
                                   .s3Client(s3Async)
                                   .build();
+        tmNonMultipartJava = S3TransferManager.builder()
+                                              .s3Client(s3NonMultipartAsync)
+                                              .build();
 
     }
 
     @AfterAll
     public static void cleanUpForAllIntegTests() {
         s3.close();
         s3Async.close();
+        s3NonMultipartAsync.close();
         s3CrtAsync.close();
         tmCrt.close();
+        tmJava.close();
+        tmNonMultipartJava.close();
         CrtResource.waitForNoResources();
     }
 
@@ -182,4 +195,11 @@ static Stream<Arguments> transferManagers() {
             Arguments.of(tmJava));
     }
 
+    static Stream<Arguments> presignedUrlTransferManagers() {
+        return Stream.of(
+            // TODO: uncomment when CRT for presigned URL is supported
+            // Arguments.of(tmCrt),
+            Arguments.of(tmNonMultipartJava),
+            Arguments.of(tmJava));
+    }
 }

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
+import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest;
+import software.amazon.awssdk.testutils.RandomTempFile;
+import software.amazon.awssdk.transfer.s3.model.CompletedDownload;
+import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
+import software.amazon.awssdk.transfer.s3.model.FileDownload;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest;
+import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;
+import software.amazon.awssdk.utils.Md5Utils;
+
+public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase {
+    private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class);
+    private static final String SMALL_KEY = "small-key";
+    private static final String LARGE_KEY = "large-key";
+    private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024;
+    private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024;
+
+    private static File smallFile;
+    private static File largeFile;
+    private static S3Presigner presigner;
+
+    @BeforeAll
+    public static void setup() throws IOException {
+        createBucket(BUCKET);
+        smallFile = new RandomTempFile(SMALL_OBJ_SIZE);
+        largeFile = new RandomTempFile(LARGE_OBJ_SIZE);
+        s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath());
+        s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath());
+        presigner = S3Presigner.builder()
+                              .region(DEFAULT_REGION)
+                              .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
+                              .build();
+    }
+
+    @AfterAll
+    public static void cleanup() {
+        if (presigner != null) {
+            presigner.close();
+        }
+        deleteBucketAndAllContents(BUCKET);
+    }
+
+    static Stream<Arguments> testCases() {
+        return presignedUrlTransferManagers().flatMap(tmArg -> Stream.of(
+            Arguments.of(tmArg.get()[0], SMALL_KEY, smallFile, SMALL_OBJ_SIZE),
+            Arguments.of(tmArg.get()[0], LARGE_KEY, largeFile, LARGE_OBJ_SIZE)
+        ));
+    }
+
+    @ParameterizedTest(name = "downloadFileWithPresignedUrl_{1}")
+    @MethodSource("testCases")
+    void downloadFileWithPresignedUrl_shouldDownloadCorrectly(S3TransferManager tm, String key,
+                                                              File sourceFile, int objSize) throws Exception {
+        Path downloadPath = RandomTempFile.randomUncreatedFile().toPath();
+        FileDownload download = tm.downloadFileWithPresignedUrl(createFileDownloadRequest(key, downloadPath));
+        CompletedFileDownload completed = download.completionFuture().join();
+
+        assertThat(Files.exists(downloadPath)).isTrue();
+        assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(sourceFile));
+        assertThat(completed.response().responseMetadata().requestId()).isNotNull();
+    }
+
+    @ParameterizedTest(name = "downloadWithPresignedUrl_toBytes_{1}")
+    @MethodSource("testCases")
+    void downloadWithPresignedUrl_toBytes_shouldReturnCorrectData(S3TransferManager tm, String key,
+                                                                   File sourceFile, int objSize) {
+        CompletedDownload<ResponseBytes<GetObjectResponse>> completed =
+            tm.downloadWithPresignedUrl(createBytesDownloadRequest(key)).completionFuture().join();
+
+        assertThat(completed.result().asByteArray()).hasSize(objSize);
+    }
+
+    private static PresignedDownloadFileRequest createFileDownloadRequest(String key, Path destination) {
+        return PresignedDownloadFileRequest.builder()
+            .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
+                .presignedUrl(createPresignedRequest(key).url())
+                .build())
+            .destination(destination)
+            .addTransferListener(LoggingTransferListener.create())
+            .build();
+    }
+
+    private static PresignedDownloadRequest<ResponseBytes<GetObjectResponse>> createBytesDownloadRequest(String key) {
+        return PresignedDownloadRequest.builder()
+            .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder()
+                .presignedUrl(createPresignedRequest(key).url())
+                .build())
+            .responseTransformer(AsyncResponseTransformer.toBytes())
+            .addTransferListener(LoggingTransferListener.create())
+            .build();
+    }
+
+    private static PresignedGetObjectRequest createPresignedRequest(String key) {
+        return presigner.presignGetObject(GetObjectPresignRequest.builder()
+            .signatureDuration(Duration.ofMinutes(10))
+            .getObjectRequest(GetObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(key)
+                .build())
+            .build());
+    }
+}

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java

@@ -41,6 +41,8 @@
 import software.amazon.awssdk.transfer.s3.model.DownloadRequest;
 import software.amazon.awssdk.transfer.s3.model.FileDownload;
 import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest;
 import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
 import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload;
 import software.amazon.awssdk.transfer.s3.model.Upload;
@@ -696,6 +698,107 @@ default Copy copy(Consumer<CopyRequest.Builder> copyRequestBuilder) {
         return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build());
     }
 
+    /**
+     * Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use
+     * {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead.
+     * <p>
+     * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
+     * providing enhanced throughput and reliability for large objects. Progress can be monitored
+     * through {@link TransferListener}s attached to the request.
+     * <p>
+     * The SDK will create a new file if the provided destination doesn't exist. If the file already exists,
+     * it will be replaced. In the event of an error, the SDK will <b>NOT</b> attempt to delete
+     * the file, leaving it as-is.
+     * <p>
+     * Note: The result of the operation doesn't support pause and resume functionality.
+     * </p>
+     * <p>
+     * <b>Usage Example:</b>
+     * {@snippet :
+     *         S3TransferManager transferManager = S3TransferManager.create();
+     *         
+     *         // Create presigned URL (typically done by another service)
+     *         PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
+     *                                                                                  .presignedUrl(presignedUrl)
+     *                                                                                  .build();
+     *         
+     *         PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder()
+     *                                                                            .presignedUrlDownloadRequest(presignedRequest)
+     *                                                                            .destination(Paths.get("downloaded-file.txt"))
+     *                                                                            .addTransferListener(
+     *                                                                                    LoggingTransferListener.create())
+     *                                                                            .build();
+     *         
+     *         FileDownload download = transferManager.downloadFileWithPresignedUrl(request);
+     *         download.completionFuture().join();
+     * }
+     *
+     * @param presignedDownloadFileRequest the presigned download file request
+     * @return A {@link FileDownload} that can be used to track the ongoing transfer
+     * @see #downloadFileWithPresignedUrl(Consumer)
+     * @see #downloadWithPresignedUrl(PresignedDownloadRequest)
+     */
+    default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder,
+     * avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}.
+     * <p>
+     * Note: The result of the operation doesn't support pause and resume functionality.
+     * </p>
+     *
+     * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
+     */
+    default FileDownload downloadFileWithPresignedUrl(
+            Consumer<PresignedDownloadFileRequest.Builder> presignedDownloadFileRequest) {
+        return downloadFileWithPresignedUrl(
+                PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build());
+    }
+
+    /**
+     * Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading
+     * to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead.
+     * <p>
+     * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client,
+     * providing enhanced throughput and reliability for large objects. Progress can be monitored
+     * through {@link TransferListener}s attached to the request.
+     * <p>
+     * Note: The result of the operation doesn't support pause and resume functionality.
+     * </p>
+     * <p>
+     * <b>Usage Example (downloading to memory - not suitable for large objects):</b>
+     * {@snippet :
+     *         S3TransferManager transferManager = S3TransferManager.create();
+     *         
+     *         // Create presigned URL (typically done by another service)
+     *         PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder()
+     *                                                                                  .presignedUrl(presignedUrl)
+     *                                                                                  .build();
+     *         
+     *         PresignedDownloadRequest<ResponseBytes<GetObjectResponse>> request = 
+     *             PresignedDownloadRequest.builder()
+     *                                    .presignedUrlDownloadRequest(presignedRequest)
+     *                                    .responseTransformer(AsyncResponseTransformer.toBytes())
+     *                                    .addTransferListener(LoggingTransferListener.create())
+     *                                    .build();
+     *         
+     *         Download<ResponseBytes<GetObjectResponse>> download = transferManager.downloadWithPresignedUrl(request);
+     *         ResponseBytes<GetObjectResponse> result = download.completionFuture().join().result();
+     * }
+     *
+     * @param presignedDownloadRequest the presigned download request
+     * @param <ResultT> The type of data the {@link AsyncResponseTransformer} produces
+     * @return A {@link Download} that can be used to track the ongoing transfer
+     * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)
+     * @see AsyncResponseTransformer
+     */
+    default <ResultT> Download<ResultT> downloadWithPresignedUrl(
+            PresignedDownloadRequest<ResultT> presignedDownloadRequest) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * Create an {@code S3TransferManager} using the default values.
      * <p>

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java

@@ -27,6 +27,8 @@
 import software.amazon.awssdk.transfer.s3.model.DownloadRequest;
 import software.amazon.awssdk.transfer.s3.model.FileDownload;
 import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest;
+import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest;
 import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
 import software.amazon.awssdk.transfer.s3.model.Upload;
 import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest;
@@ -85,6 +87,16 @@ public Copy copy(CopyRequest copyRequest) {
         return delegate.copy(copyRequest);
     }
 
+    @Override
+    public FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) {
+        return delegate.downloadFileWithPresignedUrl(presignedDownloadFileRequest);
+    }
+
+    @Override
+    public <ResultT> Download<ResultT> downloadWithPresignedUrl(PresignedDownloadRequest<ResultT> presignedDownloadRequest) {
+        return delegate.downloadWithPresignedUrl(presignedDownloadRequest);
+    }
+
     @Override
     public void close() {
         delegate.close();