From 995ca7d0112b0e1f73d058c8abf0e4b54228744a Mon Sep 17 00:00:00 2001 From: Bennett Lynch Date: Tue, 2 Nov 2021 21:12:11 -0700 Subject: [PATCH 1/4] Refactor S3TransferManager to support non-file-based transfers ## Motivation We would like for S3TransferManager to support non-file-based uploads and downloads, such as reactive streams or in-memory byte buffers, but the current implementation is directly coupled to file/Path-based transfers. While the underlying S3AsyncClient already has support for AsyncRequestBodies and AsyncResponseTransformers, these constructs are not exposed through the current S3TransferManager interface. Additionally, the existing S3TransferManager interfaces have made some assumptions that make it difficult to introduce this support, such as UploadRequest/DownloadRequest having a Path attribute, and not having interfaces that allow us to distinguish between transfer types, e.g., single-object and multi-object (directory) transfers. Likewise, the current interface hierarchy also makes it difficult to distinguish between file-based and non-file-based. Therefore, this refactoring requires backwards-incompatible changes, which are permissible while S3TransferManager is still in PREVIEW. ## Modifications The diff here is very large, but this change set primarily introduces the following interface hierarchy: * TransferRequest * TransferObjectRequest * UploadFileRequest * DownloadFileRequest * UploadRequest * DownloadRequest * TransferDirectoryRequest * UploadDirectoryRequest * Transfer * ObjectTransfer * FileUpload * FileDownload * Upload * Download * DirectoryTransfer * DirectoryUpload * CompletedTransfer * CompletedObjectTransfer * CompletedFileUpload * CompletedFileDownload * CompletedUpload * CompletedDownload * CompletedDirectoryTransfer * CompletedDirectoryUpload * FailedObjectTransfer * FailedFileUpload These interfaces allow us to more selectively choose which attributes will be shared between different data types. Additionally, we take this opportunity to make the naming convention and class-vs-interface distinction consistent across all the above data types. It is important that we distinguish between file-based-requests and non-file-based-requests so that the Collection<> of failed file transfers from a directory upload can be re-driven, if needed. Note that the above Download-oriented data types are all generic. This is because of how AsyncResponseTransformer is designed to be generic, being parameterized with the type of data that a given AsyncResponseTransformer produces. Therefore, a user must declare the type at the time of instantiating a DownloadRequest, and the type will be consistent throughout the transfer lifecycle, including the Download and CompletedDownload interfaces. --- .../feature-AmazonS3-2a00881.json | 6 + ...xceptionTransformationIntegrationTest.java | 37 ++- ...ransferManagerDownloadIntegrationTest.java | 36 ++- ...ManagerUploadDirectoryIntegrationTest.java | 26 +- ...3TransferManagerUploadIntegrationTest.java | 45 ++- .../s3/CompletedDirectoryTransfer.java | 56 ++++ .../transfer/s3/CompletedDirectoryUpload.java | 127 ++++++++ .../awssdk/transfer/s3/CompletedDownload.java | 136 ++++++--- .../transfer/s3/CompletedFileDownload.java | 116 ++++++++ .../transfer/s3/CompletedFileUpload.java | 115 ++++++++ .../transfer/s3/CompletedObjectTransfer.java | 32 ++ .../awssdk/transfer/s3/CompletedTransfer.java | 5 +- .../awssdk/transfer/s3/CompletedUpload.java | 13 +- .../transfer/s3/CompletedUploadDirectory.java | 146 --------- .../awssdk/transfer/s3/DirectoryTransfer.java | 29 ++ .../awssdk/transfer/s3/DirectoryUpload.java | 30 ++ .../amazon/awssdk/transfer/s3/Download.java | 12 +- .../transfer/s3/DownloadFileRequest.java | 274 +++++++++++++++++ .../awssdk/transfer/s3/DownloadRequest.java | 272 +++++++++++------ .../awssdk/transfer/s3/FailedFileUpload.java | 46 ++- ...ransfer.java => FailedObjectTransfer.java} | 31 +- .../awssdk/transfer/s3/FileDownload.java | 31 ++ .../amazon/awssdk/transfer/s3/FileUpload.java | 31 ++ .../awssdk/transfer/s3/ObjectTransfer.java | 37 +++ .../awssdk/transfer/s3/S3TransferManager.java | 222 +++++++------- .../amazon/awssdk/transfer/s3/Transfer.java | 3 + .../transfer/s3/TransferDirectoryRequest.java | 29 ++ .../transfer/s3/TransferObjectRequest.java | 35 +++ .../awssdk/transfer/s3/TransferRequest.java | 10 +- .../TransferRequestOverrideConfiguration.java | 4 +- .../amazon/awssdk/transfer/s3/Upload.java | 8 +- .../transfer/s3/UploadDirectoryRequest.java | 30 +- .../transfer/s3/UploadDirectoryTransfer.java | 83 ------ .../awssdk/transfer/s3/UploadFileRequest.java | 276 ++++++++++++++++++ .../awssdk/transfer/s3/UploadRequest.java | 114 ++++---- .../s3/internal/DefaultDirectoryUpload.java | 66 +++++ .../transfer/s3/internal/DefaultDownload.java | 43 ++- .../s3/internal/DefaultFileDownload.java | 80 +++++ .../s3/internal/DefaultFileUpload.java | 78 +++++ .../s3/internal/DefaultS3TransferManager.java | 115 ++++++-- .../transfer/s3/internal/DefaultUpload.java | 37 ++- .../s3/internal/UploadDirectoryHelper.java | 68 ++--- .../progress/TransferListenerContext.java | 22 +- .../TransferListenerFailedContext.java | 4 +- .../progress/TransferProgressUpdater.java | 30 +- .../s3/progress/TransferListener.java | 31 +- .../s3/progress/TransferProgress.java | 4 +- .../s3/progress/TransferProgressSnapshot.java | 6 +- .../transfer/s3/CompletedDownloadTest.java | 6 +- .../s3/CompletedFileDownloadTest.java | 37 +++ .../transfer/s3/CompletedFileUploadTest.java | 37 +++ .../transfer/s3/CompletedUploadTest.java | 2 +- .../transfer/s3/DownloadFileRequestTest.java | 82 ++++++ .../transfer/s3/DownloadRequestTest.java | 37 +-- ...oadTest.java => FailedFileUploadTest.java} | 8 +- .../transfer/s3/UploadFileRequestTest.java | 80 +++++ .../awssdk/transfer/s3/UploadRequestTest.java | 27 +- ...java => CompletedDirectoryUploadTest.java} | 8 +- .../S3TransferManagerListenerTest.java | 85 +++--- .../s3/internal/S3TransferManagerTest.java | 84 +++--- ...ploadDirectoryHelperParameterizedTest.java | 68 ++--- .../internal/UploadDirectoryHelperTest.java | 76 ++--- .../progress/LoggingTransferListenerTest.java | 8 +- .../transfer/s3/util/ChecksumUtils.java | 8 + .../TransferManagerDownloadBenchmark.java | 8 +- .../TransferManagerUploadBenchmark.java | 4 +- 66 files changed, 2758 insertions(+), 994 deletions(-) create mode 100644 .changes/next-release/feature-AmazonS3-2a00881.json create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileDownload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedObjectTransfer.java delete mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryTransfer.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryUpload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFileRequest.java rename services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/{FailedFileTransfer.java => FailedObjectTransfer.java} (55%) create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileDownload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileUpload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/ObjectTransfer.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferDirectoryRequest.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferObjectRequest.java delete mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadFileRequest.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUpload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUpload.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileDownloadTest.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileUploadTest.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadFileRequestTest.java rename services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/{FailedSingleFileUploadTest.java => FailedFileUploadTest.java} (88%) create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadFileRequestTest.java rename services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/{CompletedUploadDirectoryTest.java => CompletedDirectoryUploadTest.java} (76%) diff --git a/.changes/next-release/feature-AmazonS3-2a00881.json b/.changes/next-release/feature-AmazonS3-2a00881.json new file mode 100644 index 000000000000..9a47dc81dabc --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-2a00881.json @@ -0,0 +1,6 @@ +{ + "category": "Amazon S3", + "contributor": "", + "type": "feature", + "description": "Refactor S3TransferManager to support non-file-based transfers" +} diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java index f61714e4efc4..56d3ebd099d4 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java @@ -15,6 +15,12 @@ package software.amazon.awssdk.transfer.s3; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -25,13 +31,6 @@ import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; - public class CrtExceptionTransformationIntegrationTest extends S3IntegrationTestBase { private static final String BUCKET = temporaryBucketName(CrtExceptionTransformationIntegrationTest.class); @@ -87,10 +86,10 @@ public void getObjectNoSuchBucket() throws IOException { @Test public void transferManagerDownloadObjectWithNoSuchKey() throws IOException { String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); - assertThatThrownBy(() -> transferManager.download(DownloadRequest.builder() - .getObjectRequest(GetObjectRequest.builder().bucket(BUCKET).key("randomKey").build()) - .destination(Paths.get(randomBaseDirectory).resolve("testFile")) - .build()).completionFuture().join()) + assertThatThrownBy(() -> transferManager.downloadFile(DownloadFileRequest.builder() + .getObjectRequest(GetObjectRequest.builder().bucket(BUCKET).key("randomKey").build()) + .destination(Paths.get(randomBaseDirectory).resolve("testFile")) + .build()).completionFuture().join()) .hasCauseInstanceOf(NoSuchKeyException.class) .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchKeyException: The specified key does not exist"); } @@ -98,10 +97,10 @@ public void transferManagerDownloadObjectWithNoSuchKey() throws IOException { @Test public void transferManagerDownloadObjectWithNoSuchBucket() throws IOException { String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); - assertThatThrownBy(() -> transferManager.download(DownloadRequest.builder() - .getObjectRequest(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build()) - .destination(Paths.get(randomBaseDirectory).resolve("testFile")) - .build()).completionFuture().join()) + assertThatThrownBy(() -> transferManager.downloadFile(DownloadFileRequest.builder() + .getObjectRequest(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build()) + .destination(Paths.get(randomBaseDirectory).resolve("testFile")) + .build()).completionFuture().join()) .hasCauseInstanceOf(NoSuchBucketException.class) .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); } @@ -127,10 +126,10 @@ public void putObjectNoSuchBucket() throws IOException { @Test public void transferManagerUploadObjectWithNoSuchObject() throws IOException{ - assertThatThrownBy(() -> transferManager.upload(UploadRequest.builder() - .putObjectRequest(PutObjectRequest.builder().bucket("nonExistingTestBucket").key("someKey").build()) - .source(testFile.toPath()) - .build()).completionFuture().join()) + assertThatThrownBy(() -> transferManager.uploadFile(UploadFileRequest.builder() + .putObjectRequest(PutObjectRequest.builder().bucket("nonExistingTestBucket").key("someKey").build()) + .source(testFile.toPath()) + .build()).completionFuture().join()) .hasCauseInstanceOf(NoSuchBucketException.class) .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java index 4b18c8231f25..124117637779 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java @@ -24,6 +24,9 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; @@ -46,7 +49,7 @@ public static void setup() throws IOException { .build(), file.toPath()); tm = S3TransferManager.builder() .s3ClientConfiguration(b -> b.region(DEFAULT_REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)) .build(); } @@ -58,15 +61,30 @@ public static void cleanup() { } @Test - public void download_shouldWork() throws IOException { + public void download_toFile() throws IOException { Path path = RandomTempFile.randomUncreatedFile().toPath(); - Download download = tm.download(DownloadRequest.builder() - .getObjectRequest(b -> b.bucket(BUCKET).key(KEY)) - .destination(path) - .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - .build()); - CompletedDownload completedDownload = download.completionFuture().join(); + FileDownload download = + tm.downloadFile(DownloadFileRequest.builder() + .getObjectRequest(b -> b.bucket(BUCKET).key(KEY)) + .destination(path) + .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) + .build()); + CompletedFileDownload completedFileDownload = download.completionFuture().join(); assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file)); - assertThat(completedDownload.response().responseMetadata().requestId()).isNotNull(); + assertThat(completedFileDownload.response().responseMetadata().requestId()).isNotNull(); + } + + @Test + public void download_toBytes() throws Exception { + Download> download = + tm.download(DownloadRequest.builder() + .getObjectRequest(b -> b.bucket(BUCKET).key(KEY)) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) + .build()); + CompletedDownload> completedDownload = download.completionFuture().join(); + ResponseBytes result = completedDownload.result(); + assertThat(Md5Utils.md5AsBase64(result.asByteArray())).isEqualTo(Md5Utils.md5AsBase64(file)); + assertThat(result.response().responseMetadata().requestId()).isNotNull(); } } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java index 9fa74864e7e8..b57e4812c176 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java @@ -85,12 +85,12 @@ public static void teardown() { @Test public void uploadDirectory_filesSentCorrectly() { String prefix = "yolo"; - UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory) - .bucket(TEST_BUCKET) - .prefix(prefix) - .overrideConfiguration(o -> o.recursive(true))); - CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join(); - assertThat(completedUploadDirectory.failedUploads()).isEmpty(); + DirectoryUpload uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory) + .bucket(TEST_BUCKET) + .prefix(prefix) + .overrideConfiguration(o -> o.recursive(true))); + CompletedDirectoryUpload completedDirectoryUpload = uploadDirectory.completionFuture().join(); + assertThat(completedDirectoryUpload.failedTransfers()).isEmpty(); List keys = s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key) @@ -105,13 +105,13 @@ public void uploadDirectory_filesSentCorrectly() { public void uploadDirectory_withDelimiter_filesSentCorrectly() { String prefix = "hello"; String delimiter = "0"; - UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory) - .bucket(TEST_BUCKET) - .delimiter(delimiter) - .prefix(prefix) - .overrideConfiguration(o -> o.recursive(true))); - CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join(); - assertThat(completedUploadDirectory.failedUploads()).isEmpty(); + DirectoryUpload uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory) + .bucket(TEST_BUCKET) + .delimiter(delimiter) + .prefix(prefix) + .overrideConfiguration(o -> o.recursive(true))); + CompletedDirectoryUpload completedDirectoryUpload = uploadDirectory.completionFuture().join(); + assertThat(completedDirectoryUpload.failedTransfers()).isEmpty(); List keys = s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key) diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java index f0165db0f67d..cdfa315fe5ed 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java @@ -19,11 +19,14 @@ import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.UUID; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.testutils.RandomTempFile; @@ -62,16 +65,17 @@ public static void teardown() throws IOException { } @Test - public void upload_fileSentCorrectly() throws IOException { - Upload upload = tm.upload(UploadRequest.builder() - .putObjectRequest(b -> b.bucket(TEST_BUCKET).key(TEST_KEY)) - .source(testFile.toPath()) - .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - .build()); + public void upload_file_SentCorrectly() throws IOException { + FileUpload fileUpload = + tm.uploadFile(UploadFileRequest.builder() + .putObjectRequest(b -> b.bucket(TEST_BUCKET).key(TEST_KEY)) + .source(testFile.toPath()) + .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) + .build()); - CompletedUpload completedUpload = upload.completionFuture().join(); - assertThat(completedUpload.response().responseMetadata().requestId()).isNotNull(); - assertThat(completedUpload.response().sdkHttpResponse()).isNotNull(); + CompletedFileUpload completedFileUpload = fileUpload.completionFuture().join(); + assertThat(completedFileUpload.response().responseMetadata().requestId()).isNotNull(); + assertThat(completedFileUpload.response().sdkHttpResponse()).isNotNull(); ResponseInputStream obj = s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), ResponseTransformer.toInputStream()); @@ -80,4 +84,27 @@ public void upload_fileSentCorrectly() throws IOException { .isEqualTo(ChecksumUtils.computeCheckSum(obj)); assertThat(obj.response().responseMetadata().requestId()).isNotNull(); } + + @Test + public void upload_asyncRequestBody_SentCorrectly() throws IOException { + String content = UUID.randomUUID().toString(); + + Upload upload = + tm.upload(UploadRequest.builder() + .putObjectRequest(b -> b.bucket(TEST_BUCKET).key(TEST_KEY)) + .requestBody(AsyncRequestBody.fromString(content)) + .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) + .build()); + + CompletedUpload completedUpload = upload.completionFuture().join(); + assertThat(completedUpload.response().responseMetadata().requestId()).isNotNull(); + assertThat(completedUpload.response().sdkHttpResponse()).isNotNull(); + + ResponseInputStream obj = s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); + + assertThat(ChecksumUtils.computeCheckSum(content.getBytes(StandardCharsets.UTF_8))) + .isEqualTo(ChecksumUtils.computeCheckSum(obj)); + assertThat(obj.response().responseMetadata().requestId()).isNotNull(); + } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java new file mode 100644 index 000000000000..2cee56e09214 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Collection; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed directory-based transfer. + * + * @see CompletedDirectoryUpload + */ +@SdkPublicApi +@SdkPreviewApi +public interface CompletedDirectoryTransfer extends CompletedTransfer { + + /** + * An immutable collection of failed transfers with error details, request metadata about each file that is failed to + * transfer. + * + *

+ * Failed single object transfers can be retried by calling {@link S3TransferManager#uploadFile(UploadFileRequest)} or + * {@link S3TransferManager#downloadFile(DownloadFileRequest)}. + * + * 

+     * {@code
+     * // Retrying failed uploads if the exception is retryable
+     * List> futures =
+     *     completedDirectoryUpload.failedTransfers()
+     *                             .stream()
+     *                             .filter(failedSingleFileUpload -> isRetryable(failedSingleFileUpload.exception()))
+     *                             .map(failedSingleFileUpload ->
+     *                                  tm.upload(failedSingleFileUpload.request()).completionFuture())
+     *                             .collect(Collectors.toList());
+     * CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+     * }
+     *
+ * + * @return a list of failed transfers + */ + Collection failedTransfers(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java new file mode 100644 index 000000000000..a39ffbb77812 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; + +/** + * Represents a completed upload directory transfer to Amazon S3. It can be used to track + * failed single file uploads. + * + * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest) + */ +@SdkPublicApi +@SdkPreviewApi +public final class CompletedDirectoryUpload implements CompletedDirectoryTransfer { + + private final Collection failedTransfers; + + private CompletedDirectoryUpload(DefaultBuilder builder) { + this.failedTransfers = Collections.unmodifiableCollection( + Validate.paramNotNull(builder.failedTransfers, "failedTransfers")); + } + + @Override + public Collection failedTransfers() { + return failedTransfers; + } + + /** + * Creates a default builder for {@link CompletedDirectoryUpload}. + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CompletedDirectoryUpload that = (CompletedDirectoryUpload) o; + + return Objects.equals(failedTransfers, that.failedTransfers); + } + + @Override + public int hashCode() { + return failedTransfers != null ? failedTransfers.hashCode() : 0; + } + + @Override + public String toString() { + return ToString.builder("CompletedDirectoryUpload") + .add("failedTransfers", failedTransfers) + .build(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + public interface Builder { + + /** + * Sets a collection of {@link FailedFileUpload}s + * + * @param failedTransfers failed uploads + * @return This builder for method chaining. + */ + Builder failedTransfers(Collection failedTransfers); + + /** + * Builds a {@link CompletedDirectoryUpload} based on the properties supplied to this builder + * @return An initialized {@link CompletedDirectoryUpload} + */ + CompletedDirectoryUpload build(); + } + + private static final class DefaultBuilder implements Builder { + private Collection failedTransfers = Collections.emptyList(); + + private DefaultBuilder() { + } + + @Override + public Builder failedTransfers(Collection failedTransfers) { + this.failedTransfers = failedTransfers; + return this; + } + + public Collection getFailedTransfers() { + return failedTransfers; + } + + public void setFailedTransfers(Collection failedTransfers) { + failedTransfers(failedTransfers); + } + + @Override + public CompletedDirectoryUpload build() { + return new CompletedDirectoryUpload(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java index f2ff4c81fc46..22a4d1e48748 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java @@ -15,33 +15,52 @@ package software.amazon.awssdk.transfer.s3; +import java.util.Objects; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.transfer.s3.CompletedDownload.TypedBuilder; +import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** - * Represents a completed download transfer from Amazon S3. It can be used to track - * the underlying {@link GetObjectResponse} + * Represents a completed download transfer from Amazon S3. It can be used to track the underlying result + * that was transformed via an {@link AsyncResponseTransformer}. * - * @see S3TransferManager#download(DownloadRequest) + * @see S3TransferManager#download(DownloadRequest) */ @SdkPublicApi @SdkPreviewApi -public final class CompletedDownload implements CompletedTransfer { - private final GetObjectResponse response; +public final class CompletedDownload + implements CompletedObjectTransfer, + ToCopyableBuilder, CompletedDownload> { - private CompletedDownload(DefaultBuilder builder) { - this.response = Validate.paramNotNull(builder.response, "response"); + private final ResultT result; + + private CompletedDownload(DefaultTypedBuilder builder) { + this.result = Validate.paramNotNull(builder.result, "result"); } + /** - * Returns the API response from the {@link S3TransferManager#download(DownloadRequest)} - * @return the response + * Create a builder that can be used to create a {@link CompletedDownload}. + * + * @see UntypedBuilder */ - public GetObjectResponse response() { - return response; + public static UntypedBuilder builder() { + return new DefaultUntypedBuilder(); + } + + + @Override + public TypedBuilder toBuilder() { + return new DefaultTypedBuilder<>(this); + } + + public ResultT result() { + return result; } @Override @@ -53,59 +72,96 @@ public boolean equals(Object o) { return false; } - CompletedDownload that = (CompletedDownload) o; + CompletedDownload that = (CompletedDownload) o; - return response.equals(that.response); + return Objects.equals(result, that.result); } @Override public int hashCode() { - return response.hashCode(); + return result != null ? result.hashCode() : 0; } - public static Builder builder() { - return new DefaultBuilder(); + @Override + public String toString() { + return ToString.builder("CompletedDownload") + .add("result", result) + .build(); } - public interface Builder { + /** + * Initial calls to {@link CompletedDownload#builder()} return an {@link UntypedBuilder}, where the builder is not yet + * parameterized with the generic type associated with {@link CompletedDownload}. This prevents the otherwise awkward syntax + * of having to explicitly cast the builder type, e.g., + * 
+     * {@code CompletedDownload.>builder()}
+     *
+ * Instead, the type may be inferred as part of specifying the {@link #result(Object)} parameter, at which point the builder + * chain will return a new {@link TypedBuilder}. + */ + public interface UntypedBuilder { + /** - * Specify the {@link GetObjectResponse} from {@link S3AsyncClient#getObject} + * Specifies the result of the completed download. This method also infers the generic type of {@link CompletedDownload} + * to create. * - * @param response the response - * @return This builder for method chaining. + * @param result the result of the completed download, as transformed by an {@link AsyncResponseTransformer} + * @param the type of {@link CompletedDownload} to create + * @return a reference to this object so that method calls can be chained together. */ - Builder response(GetObjectResponse response); + TypedBuilder result(T result); + } + + private static class DefaultUntypedBuilder implements UntypedBuilder { + private DefaultUntypedBuilder() { + } + + @Override + public TypedBuilder result(T result) { + return new DefaultTypedBuilder() + .result(result); + } + } + + /** + * The type-parameterized version of {@link UntypedBuilder}. This builder's type is inferred as part of specifying {@link + * #result(Object)}, after which this builder can be used to construct a {@link CompletedDownload} with the same generic + * type. + */ + public interface TypedBuilder extends CopyableBuilder, CompletedDownload> { /** - * Builds a {@link CompletedUpload} based on the properties supplied to this builder - * @return An initialized {@link CompletedUpload} + * Specifies the result of the completed download. The generic type used is constrained by the {@link + * UntypedBuilder#result(Object)} that was previously used to create this {@link TypedBuilder}. + * + * @param result the result of the completed download, as transformed by an {@link AsyncResponseTransformer} + * @return a reference to this object so that method calls can be chained together. */ - CompletedDownload build(); + TypedBuilder result(T result); } - private static final class DefaultBuilder implements Builder { - private GetObjectResponse response; - private DefaultBuilder() { - } + private static class DefaultTypedBuilder implements TypedBuilder { + private T result; - @Override - public Builder response(GetObjectResponse response) { - this.response = response; - return this; + private DefaultTypedBuilder() { } - public void setResponse(GetObjectResponse response) { - response(response); + private DefaultTypedBuilder(CompletedDownload request) { + this.result = request.result; } - public GetObjectResponse getResponse() { - return response; + + @Override + public TypedBuilder result(T result) { + this.result = result; + return this; } + @Override - public CompletedDownload build() { - return new CompletedDownload(this); + public CompletedDownload build() { + return new CompletedDownload<>(this); } } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileDownload.java new file mode 100644 index 000000000000..39ad9782a5e8 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileDownload.java @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; + +/** + * Represents a completed download transfer from Amazon S3. It can be used to track + * the underlying {@link GetObjectResponse} + * + * @see S3TransferManager#downloadFile(DownloadFileRequest) + */ +@SdkPublicApi +@SdkPreviewApi +public final class CompletedFileDownload implements CompletedObjectTransfer { + private final GetObjectResponse response; + + private CompletedFileDownload(DefaultBuilder builder) { + this.response = Validate.paramNotNull(builder.response, "response"); + } + + public GetObjectResponse response() { + return response; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CompletedFileDownload that = (CompletedFileDownload) o; + + return Objects.equals(response, that.response); + } + + @Override + public int hashCode() { + return response != null ? response.hashCode() : 0; + } + + @Override + public String toString() { + return ToString.builder("CompletedFileDownload") + .add("response", response) + .build(); + } + + public static Builder builder() { + return new DefaultBuilder(); + } + + public interface Builder { + /** + * Specify the {@link GetObjectResponse} from {@link S3AsyncClient#getObject} + * + * @param response the response + * @return This builder for method chaining. + */ + Builder response(GetObjectResponse response); + + /** + * Builds a {@link CompletedFileUpload} based on the properties supplied to this builder + * @return An initialized {@link CompletedFileDownload} + */ + CompletedFileDownload build(); + } + + private static final class DefaultBuilder implements Builder { + private GetObjectResponse response; + + private DefaultBuilder() { + } + + @Override + public Builder response(GetObjectResponse response) { + this.response = response; + return this; + } + + public void setResponse(GetObjectResponse response) { + response(response); + } + + public GetObjectResponse getResponse() { + return response; + } + + @Override + public CompletedFileDownload build() { + return new CompletedFileDownload(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java new file mode 100644 index 000000000000..88cbe2ae913b --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java @@ -0,0 +1,115 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; + +/** + * Represents a completed upload transfer to Amazon S3. It can be used to track + * the underlying {@link PutObjectResponse} + * + * @see S3TransferManager#uploadFile(UploadFileRequest) + */ +@SdkPublicApi +@SdkPreviewApi +public final class CompletedFileUpload implements CompletedObjectTransfer { + private final PutObjectResponse response; + + private CompletedFileUpload(DefaultBuilder builder) { + this.response = Validate.paramNotNull(builder.response, "response"); + } + + public PutObjectResponse response() { + return response; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CompletedFileUpload that = (CompletedFileUpload) o; + + return Objects.equals(response, that.response); + } + + @Override + public int hashCode() { + return response != null ? response.hashCode() : 0; + } + + @Override + public String toString() { + return ToString.builder("CompletedFileUpload") + .add("response", response) + .build(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + /** + * Creates a default builder for {@link CompletedFileUpload}. + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + public interface Builder { + /** + * Specify the {@link PutObjectResponse} from {@link S3AsyncClient#putObject} + * + * @param response the response + * @return This builder for method chaining. + */ + Builder response(PutObjectResponse response); + + /** + * Builds a {@link CompletedFileUpload} based on the properties supplied to this builder + * @return An initialized {@link CompletedFileUpload} + */ + CompletedFileUpload build(); + } + + private static class DefaultBuilder implements Builder { + private PutObjectResponse response; + + private DefaultBuilder() { + } + + @Override + public Builder response(PutObjectResponse response) { + this.response = response; + return this; + } + + @Override + public CompletedFileUpload build() { + return new CompletedFileUpload(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedObjectTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedObjectTransfer.java new file mode 100644 index 000000000000..ad1394219d46 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedObjectTransfer.java @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed single object transfer. + * + * @see CompletedFileUpload + * @see CompletedFileDownload + * @see CompletedUpload + * @see CompletedDownload + */ +@SdkPublicApi +@SdkPreviewApi +public interface CompletedObjectTransfer extends CompletedTransfer { +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java index 28ccbf0940ba..ebaa2cddeb70 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java @@ -19,7 +19,10 @@ import software.amazon.awssdk.annotations.SdkPublicApi; /** - * A completed transfer. + * The parent interface for all completed transfers. + * + * @see CompletedObjectTransfer + * @see CompletedDirectoryTransfer */ @SdkPublicApi @SdkPreviewApi diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java index 5b101b9db41d..c6f89a5ac951 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.transfer.s3; +import java.util.Objects; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -26,21 +27,17 @@ * Represents a completed upload transfer to Amazon S3. It can be used to track * the underlying {@link PutObjectResponse} * - * @see S3TransferManager#upload(UploadRequest) + * @see S3TransferManager#upload(UploadRequest) */ @SdkPublicApi @SdkPreviewApi -public final class CompletedUpload implements CompletedTransfer { +public final class CompletedUpload implements CompletedObjectTransfer { private final PutObjectResponse response; private CompletedUpload(DefaultBuilder builder) { this.response = Validate.paramNotNull(builder.response, "response"); } - /** - * Returns the API response from the {@link S3TransferManager#upload(UploadRequest)} - * @return the response - */ public PutObjectResponse response() { return response; } @@ -56,12 +53,12 @@ public boolean equals(Object o) { CompletedUpload that = (CompletedUpload) o; - return response.equals(that.response); + return Objects.equals(response, that.response); } @Override public int hashCode() { - return response.hashCode(); + return response != null ? response.hashCode() : 0; } @Override diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java deleted file mode 100644 index db7fe32992f0..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3; - -import java.util.Collection; -import java.util.Collections; -import software.amazon.awssdk.annotations.SdkPreviewApi; -import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.utils.ToString; -import software.amazon.awssdk.utils.Validate; - -/** - * Represents a completed upload directory transfer to Amazon S3. It can be used to track - * failed single file uploads. - * - * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest) - */ -@SdkPublicApi -@SdkPreviewApi -public final class CompletedUploadDirectory implements CompletedTransfer { - private final Collection failedUploads; - - private CompletedUploadDirectory(DefaultBuilder builder) { - this.failedUploads = Collections.unmodifiableCollection(Validate.paramNotNull(builder.failedUploads, "failedUploads")); - } - - /** - * An immutable collection of failed uploads with error details, request metadata about each file that is failed to - * upload. - * - *

- * Failed single file uploads can be retried by calling {@link S3TransferManager#upload(UploadRequest)} - * - * 

-     * {@code
-     * // Retrying failed uploads if the exception is retryable
-     * List> futures =
-     *     completedUploadDirectory.failedUploads()
-     *                             .stream()
-     *                             .filter(failedSingleFileUpload -> isRetryable(failedSingleFileUpload.exception()))
-     *                             .map(failedSingleFileUpload ->
-     *                                  tm.upload(failedSingleFileUpload.request()).completionFuture())
-     *                             .collect(Collectors.toList());
-     * CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
-     * }
-     *
- * - * @return a list of failed uploads - */ - public Collection failedUploads() { - return failedUploads; - } - - /** - * Creates a default builder for {@link CompletedUploadDirectory}. - */ - public static Builder builder() { - return new DefaultBuilder(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - CompletedUploadDirectory that = (CompletedUploadDirectory) o; - - return failedUploads.equals(that.failedUploads); - } - - @Override - public int hashCode() { - return failedUploads.hashCode(); - } - - @Override - public String toString() { - return ToString.builder("CompletedUploadDirectory") - .add("failedUploads", failedUploads) - .build(); - } - - public static Class serializableBuilderClass() { - return DefaultBuilder.class; - } - - public interface Builder { - - /** - * Sets a collection of {@link FailedFileUpload}s - * - * @param failedUploads failed uploads - * @return This builder for method chaining. - */ - Builder failedUploads(Collection failedUploads); - - /** - * Builds a {@link CompletedUploadDirectory} based on the properties supplied to this builder - * @return An initialized {@link CompletedUploadDirectory} - */ - CompletedUploadDirectory build(); - } - - private static final class DefaultBuilder implements Builder { - private Collection failedUploads = Collections.emptyList(); - - private DefaultBuilder() { - } - - @Override - public Builder failedUploads(Collection failedUploads) { - this.failedUploads = failedUploads; - return this; - } - - public Collection getFailedUploads() { - return failedUploads; - } - - public void setFailedUploads(Collection failedUploads) { - failedUploads(failedUploads); - } - - @Override - public CompletedUploadDirectory build() { - return new CompletedUploadDirectory(this); - } - } -} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryTransfer.java new file mode 100644 index 000000000000..5d703b3aa159 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryTransfer.java @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Represents the upload or download of a directory of files to or from S3. + * + * @see DirectoryUpload + */ +@SdkPublicApi +@SdkPreviewApi +public interface DirectoryTransfer extends Transfer { +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryUpload.java new file mode 100644 index 000000000000..083aa8f42988 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DirectoryUpload.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * An upload transfer of a single object to S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface DirectoryUpload extends DirectoryTransfer { + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java index a9fd3e498e06..c2e9412c006a 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java @@ -18,20 +18,14 @@ import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.transfer.s3.progress.TransferProgress; /** * A download transfer of a single object from S3. */ @SdkPublicApi @SdkPreviewApi -public interface Download extends Transfer { - +public interface Download extends ObjectTransfer { + @Override - CompletableFuture completionFuture(); - - /** - * The stateful {@link TransferProgress} associated with this transfer. - */ - TransferProgress progress(); + CompletableFuture> completionFuture(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFileRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFileRequest.java new file mode 100644 index 000000000000..00feec62a743 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFileRequest.java @@ -0,0 +1,274 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.io.File; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Request object to download an object from S3 using the Transfer Manager. + */ +@SdkPublicApi +@SdkPreviewApi +public final class DownloadFileRequest + implements TransferObjectRequest, ToCopyableBuilder { + + private final Path destination; + private final GetObjectRequest getObjectRequest; + private final TransferRequestOverrideConfiguration configuration; + + private DownloadFileRequest(DefaultBuilder builder) { + this.destination = Validate.paramNotNull(builder.destination, "destination"); + this.getObjectRequest = Validate.paramNotNull(builder.getObjectRequest, "getObjectRequest"); + this.configuration = builder.configuration; + } + + /** + * Create a builder that can be used to create a {@link DownloadFileRequest}. + * + * @see S3TransferManager#downloadFile(DownloadFileRequest) + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @return the destination path + */ + public Path destination() { + return destination; + } + + /** + * @return The {@link GetObjectRequest} request that should be used for the download + */ + public GetObjectRequest getObjectRequest() { + return getObjectRequest; + } + + /** + * @return the optional override configuration + * @see Builder#overrideConfiguration(TransferRequestOverrideConfiguration) + */ + @Override + public Optional overrideConfiguration() { + return Optional.ofNullable(configuration); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DownloadFileRequest that = (DownloadFileRequest) o; + + if (!Objects.equals(destination, that.destination)) { + return false; + } + if (!Objects.equals(getObjectRequest, that.getObjectRequest)) { + return false; + } + return Objects.equals(configuration, that.configuration); + } + + @Override + public int hashCode() { + int result = destination != null ? destination.hashCode() : 0; + result = 31 * result + (getObjectRequest != null ? getObjectRequest.hashCode() : 0); + result = 31 * result + (configuration != null ? configuration.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DownloadFileRequest") + .add("destination", destination) + .add("getObjectRequest", getObjectRequest) + .add("configuration", configuration) + .build(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + /** + * A builder for a {@link DownloadFileRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends CopyableBuilder { + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder destination(Path destination); + + /** + * The file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + default Builder destination(File destination) { + Validate.paramNotNull(destination, "destination"); + return destination(destination.toPath()); + } + + /** + * The {@link GetObjectRequest} request that should be used for the download + * + * @param getObjectRequest the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(Consumer) + */ + Builder getObjectRequest(GetObjectRequest getObjectRequest); + + /** + * The {@link GetObjectRequest} request that should be used for the download + * + *

+ * This is a convenience method that creates an instance of the {@link GetObjectRequest} builder avoiding the + * need to create one manually via {@link GetObjectRequest#builder()}. + * + * @param getObjectRequestBuilder the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(GetObjectRequest) + */ + default Builder getObjectRequest(Consumer getObjectRequestBuilder) { + GetObjectRequest request = GetObjectRequest.builder() + .applyMutation(getObjectRequestBuilder) + .build(); + getObjectRequest(request); + return this; + } + + /** + * Add an optional request override configuration. + * + * @param configuration The override configuration. + * @return This builder for method chaining. + */ + Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration); + + /** + * Similar to {@link #overrideConfiguration(TransferRequestOverrideConfiguration)}, but takes a lambda to configure a new + * {@link TransferRequestOverrideConfiguration.Builder}. This removes the need to call + * {@link TransferRequestOverrideConfiguration#builder()} and + * {@link TransferRequestOverrideConfiguration.Builder#build()}. + * + * @param configurationBuilder the upload configuration + * @return this builder for method chaining. + * @see #overrideConfiguration(TransferRequestOverrideConfiguration) + */ + default Builder overrideConfiguration(Consumer configurationBuilder) { + Validate.paramNotNull(configurationBuilder, "configurationBuilder"); + return overrideConfiguration(TransferRequestOverrideConfiguration.builder() + .applyMutation(configurationBuilder) + .build()); + } + } + + private static final class DefaultBuilder implements Builder { + private Path destination; + private GetObjectRequest getObjectRequest; + private TransferRequestOverrideConfiguration configuration; + + private DefaultBuilder() { + } + + private DefaultBuilder(DownloadFileRequest downloadFileRequest) { + this.destination = downloadFileRequest.destination; + this.getObjectRequest = downloadFileRequest.getObjectRequest; + this.configuration = downloadFileRequest.configuration; + } + + @Override + public Builder destination(Path destination) { + this.destination = Validate.paramNotNull(destination, "destination"); + return this; + } + + public Path getDestination() { + return destination; + } + + public void setDestination(Path destination) { + destination(destination); + } + + @Override + public Builder getObjectRequest(GetObjectRequest getObjectRequest) { + this.getObjectRequest = getObjectRequest; + return this; + } + + public GetObjectRequest getGetObjectRequest() { + return getObjectRequest; + } + + public void setGetObjectRequest(GetObjectRequest getObjectRequest) { + getObjectRequest(getObjectRequest); + } + + @Override + public Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration) { + this.configuration = configuration; + return this; + } + + public void setOverrideConfiguration(TransferRequestOverrideConfiguration configuration) { + overrideConfiguration(configuration); + } + + public TransferRequestOverrideConfiguration getOverrideConfiguration() { + return configuration; + } + + @Override + public DownloadFileRequest build() { + return new DownloadFileRequest(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java index 6b85372b8e66..aaca423675d3 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java @@ -15,18 +15,21 @@ package software.amazon.awssdk.transfer.s3; -import java.io.File; import java.nio.file.Path; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; -import software.amazon.awssdk.annotations.NotThreadSafe; +import java.util.function.Function; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.transfer.s3.DownloadRequest.TypedBuilder; import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.SdkBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** @@ -34,13 +37,16 @@ */ @SdkPublicApi @SdkPreviewApi -public final class DownloadRequest implements TransferRequest, ToCopyableBuilder { - private final Path destination; +public final class DownloadRequest + implements TransferObjectRequest, + ToCopyableBuilder, DownloadRequest> { + + private final AsyncResponseTransformer responseTransformer; private final GetObjectRequest getObjectRequest; private final TransferRequestOverrideConfiguration overrideConfiguration; - private DownloadRequest(BuilderImpl builder) { - this.destination = Validate.paramNotNull(builder.destination, "destination"); + private DownloadRequest(DefaultTypedBuilder builder) { + this.responseTransformer = Validate.paramNotNull(builder.responseTransformer, "responseTransformer"); this.getObjectRequest = Validate.paramNotNull(builder.getObjectRequest, "getObjectRequest"); this.overrideConfiguration = builder.configuration; } @@ -48,25 +54,26 @@ private DownloadRequest(BuilderImpl builder) { /** * Create a builder that can be used to create a {@link DownloadRequest}. * - * @see S3TransferManager#download(DownloadRequest) + * @see UntypedBuilder */ - public static Builder builder() { - return new BuilderImpl(); + public static UntypedBuilder builder() { + return new DefaultUntypedBuilder(); } + @Override - public Builder toBuilder() { - return new BuilderImpl(); + public TypedBuilder toBuilder() { + return new DefaultTypedBuilder<>(this); } /** - * The {@link Path} to file that response contents will be written to. The file must not exist or this method - * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * The {@link Path} to file that response contents will be written to. The file must not exist or this method will throw an + * exception. If the file is not writable by the current user then an exception will be thrown. * * @return the destination path */ - public Path destination() { - return destination; + public AsyncResponseTransformer responseTransformer() { + return responseTransformer; } /** @@ -78,21 +85,13 @@ public GetObjectRequest getObjectRequest() { /** * @return the optional override configuration - * @see Builder#overrideConfiguration(TransferRequestOverrideConfiguration) + * @see TypedBuilder#overrideConfiguration(TransferRequestOverrideConfiguration) */ + @Override public Optional overrideConfiguration() { return Optional.ofNullable(overrideConfiguration); } - @Override - public String toString() { - return ToString.builder("DownloadRequest") - .add("destination", destination) - .add("getObjectRequest", getObjectRequest) - .add("overrideConfiguration", overrideConfiguration) - .build(); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -102,9 +101,9 @@ public boolean equals(Object o) { return false; } - DownloadRequest that = (DownloadRequest) o; + DownloadRequest that = (DownloadRequest) o; - if (!Objects.equals(destination, that.destination)) { + if (!Objects.equals(responseTransformer, that.responseTransformer)) { return false; } if (!Objects.equals(getObjectRequest, that.getObjectRequest)) { @@ -115,44 +114,32 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = destination != null ? destination.hashCode() : 0; + int result = responseTransformer != null ? responseTransformer.hashCode() : 0; result = 31 * result + (getObjectRequest != null ? getObjectRequest.hashCode() : 0); result = 31 * result + (overrideConfiguration != null ? overrideConfiguration.hashCode() : 0); return result; } - public static Class serializableBuilderClass() { - return BuilderImpl.class; + @Override + public String toString() { + return ToString.builder("DownloadRequest") + .add("responseTransformer", responseTransformer) + .add("getObjectRequest", getObjectRequest) + .add("overrideConfiguration", overrideConfiguration) + .build(); } /** - * A builder for a {@link DownloadRequest}, created with {@link #builder()} + * Initial calls to {@link DownloadRequest#builder()} return an {@link UntypedBuilder}, where the builder is not yet + * parameterized with the generic type associated with {@link DownloadRequest}. This prevents the otherwise awkward syntax of + * having to explicitly cast the builder type, e.g., + * 

+     * {@code DownloadRequest.>builder()}
+     *
+ * Instead, the type may be inferred as part of specifying the {@link #responseTransformer(AsyncResponseTransformer)} + * parameter, at which point the builder chain will return a new {@link TypedBuilder}. */ - @SdkPublicApi - @NotThreadSafe - public interface Builder extends TransferRequest.Builder, CopyableBuilder { - - /** - * The {@link Path} to file that response contents will be written to. The file must not exist or this method - * will throw an exception. If the file is not writable by the current user then an exception will be thrown. - * - * @param destination the destination path - * @return Returns a reference to this object so that method calls can be chained together. - */ - Builder destination(Path destination); - - /** - * The file that response contents will be written to. The file must not exist or this method - * will throw an exception. If the file is not writable by the current user then an exception will be thrown. - * - * @param destination the destination path - * @return Returns a reference to this object so that method calls can be chained together. - */ - default Builder destination(File destination) { - Validate.paramNotNull(destination, "destination"); - return destination(destination.toPath()); - } + public interface UntypedBuilder { /** * The {@link GetObjectRequest} request that should be used for the download @@ -161,20 +148,19 @@ default Builder destination(File destination) { * @return a reference to this object so that method calls can be chained together. * @see #getObjectRequest(Consumer) */ - Builder getObjectRequest(GetObjectRequest getObjectRequest); + UntypedBuilder getObjectRequest(GetObjectRequest getObjectRequest); /** * The {@link GetObjectRequest} request that should be used for the download - * *

- * This is a convenience method that creates an instance of the {@link GetObjectRequest} builder avoiding the - * need to create one manually via {@link GetObjectRequest#builder()}. + * This is a convenience method that creates an instance of the {@link GetObjectRequest} builder, avoiding the need to + * create one manually via {@link GetObjectRequest#builder()}. * * @param getObjectRequestBuilder the getObject request * @return a reference to this object so that method calls can be chained together. * @see #getObjectRequest(GetObjectRequest) */ - default Builder getObjectRequest(Consumer getObjectRequestBuilder) { + default UntypedBuilder getObjectRequest(Consumer getObjectRequestBuilder) { GetObjectRequest request = GetObjectRequest.builder() .applyMutation(getObjectRequestBuilder) .build(); @@ -188,85 +174,183 @@ default Builder getObjectRequest(Consumer getObjectReq * @param configuration The override configuration. * @return This builder for method chaining. */ - Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration); + UntypedBuilder overrideConfiguration(TransferRequestOverrideConfiguration configuration); /** * Similar to {@link #overrideConfiguration(TransferRequestOverrideConfiguration)}, but takes a lambda to configure a new - * {@link TransferRequestOverrideConfiguration.Builder}. This removes the need to call - * {@link TransferRequestOverrideConfiguration#builder()} and - * {@link TransferRequestOverrideConfiguration.Builder#build()}. + * {@link TransferRequestOverrideConfiguration.Builder}. This removes the need to call {@link + * TransferRequestOverrideConfiguration#builder()} and {@link TransferRequestOverrideConfiguration.Builder#build()}. * * @param configurationBuilder the upload configuration * @return this builder for method chaining. * @see #overrideConfiguration(TransferRequestOverrideConfiguration) */ - default Builder overrideConfiguration(Consumer configurationBuilder) { + default UntypedBuilder overrideConfiguration( + Consumer configurationBuilder) { Validate.paramNotNull(configurationBuilder, "configurationBuilder"); return overrideConfiguration(TransferRequestOverrideConfiguration.builder() .applyMutation(configurationBuilder) .build()); } - + /** - * @return The built request. + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. This method also infers the + * generic type of {@link DownloadRequest} to create, inferred from the second type parameter of the provided {@link + * AsyncResponseTransformer}. E.g, specifying {@link AsyncResponseTransformer#toBytes()} would result in inferring the + * type of the {@link DownloadRequest} to be of {@code ResponseBytes}. See the static factory methods + * available in {@link AsyncResponseTransformer}. + * + * @param responseTransformer the AsyncResponseTransformer + * @param the type of {@link DownloadRequest} to create + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer */ - @Override - DownloadRequest build(); + TypedBuilder responseTransformer(AsyncResponseTransformer responseTransformer); + + /** + * Like {@link SdkBuilder#applyMutation(Consumer)}, but accepts a {@link Function} that upgrades this {@link + * UntypedBuilder} to a {@link TypedBuilder}. Therefore, the function must also call {@link + * #responseTransformer(AsyncResponseTransformer)} to specify the generic type. + */ + default TypedBuilder applyMutation(Function> mutator) { + return mutator.apply(this); + } } - private static final class BuilderImpl implements Builder { - private Path destination; + private static class DefaultUntypedBuilder implements UntypedBuilder { private GetObjectRequest getObjectRequest; private TransferRequestOverrideConfiguration configuration; - private BuilderImpl() { + private DefaultUntypedBuilder() { } @Override - public Builder destination(Path destination) { - this.destination = destination; + public UntypedBuilder getObjectRequest(GetObjectRequest getObjectRequest) { + this.getObjectRequest = getObjectRequest; return this; } - public Path getDestination() { - return destination; + @Override + public UntypedBuilder overrideConfiguration(TransferRequestOverrideConfiguration configuration) { + this.configuration = configuration; + return this; } - public void setDestination(Path destination) { - destination(destination); + @Override + public TypedBuilder responseTransformer(AsyncResponseTransformer responseTransformer) { + return new DefaultTypedBuilder() + .getObjectRequest(getObjectRequest) + .overrideConfiguration(configuration) + .responseTransformer(responseTransformer); } + } - @Override - public Builder getObjectRequest(GetObjectRequest getObjectRequest) { - this.getObjectRequest = getObjectRequest; + /** + * The type-parameterized version of {@link UntypedBuilder}. This builder's type is inferred as part of specifying {@link + * UntypedBuilder#responseTransformer(AsyncResponseTransformer)}, after which this builder can be used to construct a {@link + * DownloadRequest} with the same generic type. + */ + public interface TypedBuilder extends CopyableBuilder, DownloadRequest> { + + /** + * The {@link GetObjectRequest} request that should be used for the download + * + * @param getObjectRequest the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(Consumer) + */ + TypedBuilder getObjectRequest(GetObjectRequest getObjectRequest); + + /** + * The {@link GetObjectRequest} request that should be used for the download + *

+ * This is a convenience method that creates an instance of the {@link GetObjectRequest} builder, avoiding the need to + * create one manually via {@link GetObjectRequest#builder()}. + * + * @param getObjectRequestBuilder the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(GetObjectRequest) + */ + default TypedBuilder getObjectRequest(Consumer getObjectRequestBuilder) { + GetObjectRequest request = GetObjectRequest.builder() + .applyMutation(getObjectRequestBuilder) + .build(); + getObjectRequest(request); return this; } - public GetObjectRequest getGetObjectRequest() { - return getObjectRequest; + /** + * Add an optional request override configuration. + * + * @param configuration The override configuration. + * @return This builder for method chaining. + */ + TypedBuilder overrideConfiguration(TransferRequestOverrideConfiguration configuration); + + /** + * Similar to {@link #overrideConfiguration(TransferRequestOverrideConfiguration)}, but takes a lambda to configure a new + * {@link TransferRequestOverrideConfiguration.Builder}. This removes the need to call {@link + * TransferRequestOverrideConfiguration#builder()} and {@link TransferRequestOverrideConfiguration.Builder#build()}. + * + * @param configurationBuilder the upload configuration + * @return this builder for method chaining. + * @see #overrideConfiguration(TransferRequestOverrideConfiguration) + */ + default TypedBuilder overrideConfiguration( + Consumer configurationBuilder) { + Validate.paramNotNull(configurationBuilder, "configurationBuilder"); + return overrideConfiguration(TransferRequestOverrideConfiguration.builder() + .applyMutation(configurationBuilder) + .build()); + } + + /** + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. The generic type used is + * constrained by the {@link UntypedBuilder#responseTransformer(AsyncResponseTransformer)} that was previously used to + * create this {@link TypedBuilder}. + * + * @param responseTransformer the AsyncResponseTransformer + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer + */ + TypedBuilder responseTransformer(AsyncResponseTransformer responseTransformer); + } + + private static class DefaultTypedBuilder implements TypedBuilder { + private GetObjectRequest getObjectRequest; + private TransferRequestOverrideConfiguration configuration; + private AsyncResponseTransformer responseTransformer; + + private DefaultTypedBuilder() { } - public void setGetObjectRequest(GetObjectRequest getObjectRequest) { - getObjectRequest(getObjectRequest); + private DefaultTypedBuilder(DownloadRequest request) { + this.getObjectRequest = request.getObjectRequest; + this.responseTransformer = request.responseTransformer; } @Override - public Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration) { - this.configuration = configuration; + public TypedBuilder getObjectRequest(GetObjectRequest getObjectRequest) { + this.getObjectRequest = getObjectRequest; return this; } - public void setOverrideConfiguration(TransferRequestOverrideConfiguration configuration) { - overrideConfiguration(configuration); + @Override + public DefaultTypedBuilder overrideConfiguration(TransferRequestOverrideConfiguration configuration) { + this.configuration = configuration; + return this; } - public TransferRequestOverrideConfiguration getOverrideConfiguration() { - return configuration; + @Override + public TypedBuilder responseTransformer(AsyncResponseTransformer responseTransformer) { + this.responseTransformer = responseTransformer; + return this; } + @Override - public DownloadRequest build() { - return new DownloadRequest(this); + public DownloadRequest build() { + return new DownloadRequest<>(this); } } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java index 75dc86a7fff4..0587bda6620b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.transfer.s3; +import java.util.Objects; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.utils.ToString; @@ -24,17 +25,18 @@ /** * Represents a failed single file upload from {@link S3TransferManager#uploadDirectory}. It - * has detailed description of the result + * has a detailed description of the result. */ @SdkPublicApi @SdkPreviewApi -public final class FailedFileUpload implements FailedFileTransfer, - ToCopyableBuilder { +public final class FailedFileUpload + implements FailedObjectTransfer, + ToCopyableBuilder { + + private final UploadFileRequest request; private final Throwable exception; - private final UploadRequest request; - FailedFileUpload(DefaultBuilder builder) { + private FailedFileUpload(DefaultBuilder builder) { this.exception = Validate.paramNotNull(builder.exception, "exception"); this.request = Validate.paramNotNull(builder.request, "request"); } @@ -45,7 +47,7 @@ public Throwable exception() { } @Override - public UploadRequest request() { + public UploadFileRequest request() { return request; } @@ -60,24 +62,24 @@ public boolean equals(Object o) { FailedFileUpload that = (FailedFileUpload) o; - if (!exception.equals(that.exception)) { + if (!Objects.equals(request, that.request)) { return false; } - return request.equals(that.request); + return Objects.equals(exception, that.exception); } @Override public int hashCode() { - int result = exception.hashCode(); - result = 31 * result + request.hashCode(); + int result = request != null ? request.hashCode() : 0; + result = 31 * result + (exception != null ? exception.hashCode() : 0); return result; } @Override public String toString() { - return ToString.builder("FailedUpload") - .add("exception", exception) + return ToString.builder("FailedFileUpload") .add("request", request) + .add("exception", exception) .build(); } @@ -94,21 +96,15 @@ public Builder toBuilder() { return new DefaultBuilder(this); } - public interface Builder extends CopyableBuilder, - FailedFileTransfer.Builder { + public interface Builder extends CopyableBuilder { - @Override Builder exception(Throwable exception); - @Override - Builder request(UploadRequest request); - - @Override - FailedFileUpload build(); + Builder request(UploadFileRequest request); } private static final class DefaultBuilder implements Builder { - private UploadRequest request; + private UploadFileRequest request; private Throwable exception; private DefaultBuilder(FailedFileUpload failedSingleFileUpload) { @@ -134,16 +130,16 @@ public Throwable getException() { } @Override - public Builder request(UploadRequest request) { + public Builder request(UploadFileRequest request) { this.request = request; return this; } - public void setRequest(UploadRequest request) { + public void setRequest(UploadFileRequest request) { request(request); } - public UploadRequest getRequest() { + public UploadFileRequest getRequest() { return request; } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedObjectTransfer.java similarity index 55% rename from services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileTransfer.java rename to services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedObjectTransfer.java index dcb240b15954..e19f6c74870b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileTransfer.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedObjectTransfer.java @@ -24,7 +24,7 @@ */ @SdkPublicApi @SdkPreviewApi -public interface FailedFileTransfer { +public interface FailedObjectTransfer { /** * The exception thrown from a specific single file transfer @@ -34,34 +34,9 @@ public interface FailedFileTransfer { Throwable exception(); /** - * The failed {@link TransferRequest}. + * The failed {@link TransferObjectRequest}. * * @return the failed request */ - T request(); - - interface Builder { - /** - * Specify the exception thrown from a specific single file transfer - * - * @param exception the exception thrown - * @return this builder for method chaining. - */ - Builder exception(Throwable exception); - - /** - * Specify the failed request - * - * @param request the failed request - * @return this builder for method chaining. - */ - Builder request(T request); - - /** - * Builds a {@link FailedFileTransfer} based on the properties supplied to this builder - * - * @return An initialized {@link FailedFileTransfer} - */ - FailedFileTransfer build(); - } + TransferObjectRequest request(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileDownload.java new file mode 100644 index 000000000000..578124a55e3a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileDownload.java @@ -0,0 +1,31 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A download transfer of a single object from S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface FileDownload extends ObjectTransfer { + + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileUpload.java new file mode 100644 index 000000000000..d12087377672 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FileUpload.java @@ -0,0 +1,31 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * An upload transfer of a single object to S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface FileUpload extends ObjectTransfer { + + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/ObjectTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/ObjectTransfer.java new file mode 100644 index 000000000000..a2a09c35e6eb --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/ObjectTransfer.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.transfer.s3.progress.TransferProgress; + +/** + * Represents the upload or download of a single object to or from S3. + * + * @see FileUpload + * @see FileDownload + * @see Upload + * @see Download + */ +@SdkPublicApi +@SdkPreviewApi +public interface ObjectTransfer extends Transfer { + /** + * The stateful {@link TransferProgress} associated with this transfer. + */ + TransferProgress progress(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 8c61b9ea0a6b..864b4b5febfd 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -17,8 +17,13 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +import java.util.function.Function; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager; import software.amazon.awssdk.utils.SdkAutoCloseable; import software.amazon.awssdk.utils.Validate; @@ -32,29 +37,33 @@ * 

  * {@code
  * // Create using all default configuration values
- * S3TransferManager transferManager = S3TransferManager.create();
+ * S3TransferManager tm = S3TransferManager.create();
  *
  * // If you wish to configure settings, we recommend using the builder instead:
- * S3TransferManager transferManager =
- *                  S3TransferManager.builder()
- *                                   .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider)
- *                                   .region(Region.US_WEST_2)
- *                                   .targetThroughputInGbps(20.0)
- *                                   .minimumPartSizeInBytes(10 * MB))
- *                                   .build();
+ * S3TransferManager tm =
+ *     S3TransferManager.builder()
+ *                      .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider)
+ *                                                   .region(Region.US_WEST_2)
+ *                                                   .targetThroughputInGbps(20.0)
+ *                                                   .minimumPartSizeInBytes(8 * MB))
+ *                      .build();
  *
  * // Download an S3 object to a file
- * Download download =
- *     transferManager.download(b -> b.destination(Paths.get("myFile.txt"))
- *                                    .getObjectRequest(r -> r.bucket("bucket")
- *                                                            .key("key")));
+ * FileDownload download =
+ *     tm.downloadFile(DownloadFileRequest.builder()
+ *                                        .getObjectRequest(b -> b.bucket("bucket").key("key"))
+ *                                        .destination(Paths.get("myFile.txt"))
+ *                                        .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+ *                                        .build());
  * download.completionFuture().join();
  *
  * // Upload a file to S3
- * Upload upload = transferManager.upload(b -> b.source(Paths.get("myFile.txt"))
- *                                              .putObjectRequest(r -> r.bucket("bucket")
- *                                                                      .key("key")));
- *
+ * FileUpload upload =
+ *     tm.uploadFile(UploadFileRequest.builder()
+ *                                    .putObjectRequest(b -> b.bucket("bucket"").key("key""))
+ *                                    .source(Paths.get("myFile.txt"))
+ *                                    .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+ *                                    .build());
  * upload.completionFuture().join();
  * }
  *
@@ -62,56 +71,80 @@ @SdkPublicApi @SdkPreviewApi public interface S3TransferManager extends SdkAutoCloseable { + /** - * Download an object identified by the bucket and key from S3 to the given - * file. + * Download an object identified by the bucket and key from S3 to the given file. *

* Usage Example: * 

      * {@code
      * // Initiate the transfer
-     * Download download =
-     *     transferManager.download(DownloadRequest.builder()
-     *                                             .destination(Paths.get("myFile.txt"))
-     *                                             .getObjectRequest(GetObjectRequest.builder()
-     *                                                                               .bucket("bucket")
-     *                                                                               .key("key")
-     *                                                                               .build())
-     *                                             .build());
+     * FileDownload download =
+     *     tm.downloadFile(DownloadFileRequest.builder()
+     *                                        .getObjectRequest(b -> b.bucket("bucket").key("key"))
+     *                                        .destination(Paths.get("myFile.txt"))
+     *                                        .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+     *                                        .build());
      * // Wait for the transfer to complete
      * download.completionFuture().join();
      * }
      *
- * @see #download(Consumer) + * + * @see #downloadFile(Consumer) + * @see #download(DownloadRequest) */ - default Download download(DownloadRequest downloadRequest) { + default FileDownload downloadFile(DownloadFileRequest downloadRequest) { throw new UnsupportedOperationException(); } /** - * Download an object identified by the bucket and key from S3 to the given - * file. - * - *

- * This is a convenience method that creates an instance of the {@link DownloadRequest} builder avoiding the - * need to create one manually via {@link DownloadRequest#builder()}. + * This is a convenience method that creates an instance of the {@link DownloadFileRequest} builder, avoiding the need to + * create one manually via {@link DownloadFileRequest#builder()}. * + * @see #downloadFile(DownloadFileRequest) + */ + default FileDownload downloadFile(Consumer request) { + return downloadFile(DownloadFileRequest.builder().applyMutation(request).build()); + } + + /** + * Download an object identified by the bucket and key from S3 through the given {@link AsyncResponseTransformer}. *

- * Usage Example: + * Usage Example (this example buffers the entire object into an in-memory byte array and is not suitable for + * large objects): * 

      * {@code
      * // Initiate the transfer
-     * Download download =
-     *     transferManager.download(b -> b.destination(Paths.get("myFile.txt"))
-     *                                    .getObjectRequest(r -> r.bucket("bucket")
-     *                                                            .key("key")));
+     * Download> download =
+     *     tm.download(DownloadRequest.builder()
+     *                                .getObjectRequest(b -> b.bucket("bucket"").key("key"))
+     *                                .responseTransformer(AsyncResponseTransformer.toBytes())
+     *                                .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+     *                                .build());
      * // Wait for the transfer to complete
      * download.completionFuture().join();
      * }
      *
+ * See the static factory methods available in {@link AsyncResponseTransformer} for other use cases. + * + * @param downloadRequest the download request, containing a {@link GetObjectRequest} and {@link AsyncResponseTransformer} + * @param The type of data the {@link AsyncResponseTransformer} produces + * @return A {@link Download} that can be used to track the ongoing transfer + * @see #download(Function) + * @see #downloadFile(DownloadFileRequest) + */ + default Download download(DownloadRequest downloadRequest) { + throw new UnsupportedOperationException(); + } + + /** + * This is a convenience method that creates an instance of the {@link DownloadRequest} builder, avoiding the need to create + * one manually via {@link DownloadRequest#builder()}. + * * @see #download(DownloadRequest) */ - default Download download(Consumer request) { + default Download download(Function> request) { return download(DownloadRequest.builder().applyMutation(request).build()); } @@ -121,42 +154,65 @@ default Download download(Consumer request) { * Usage Example: * 
      * {@code
-     * Upload upload =
-     *     transferManager.upload(UploadRequest.builder()
-     *                                        .source(Paths.get("myFile.txt"))
-     *                                        .putObjectRequest(PutObjectRequest.builder()
-     *                                                                          .bucket("bucket")
-     *                                                                          .key("key")
-     *                                                                          .build())
-     *                                        .build());
+     * FileUpload upload =
+     *     tm.uploadFile(UploadFileRequest.builder()
+     *                                    .putObjectRequest(b -> b.bucket("bucket"").key("key""))
+     *                                    .source(Paths.get("myFile.txt"))
+     *                                    .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+     *                                    .build());
      * // Wait for the transfer to complete
      * upload.completionFuture().join();
      * }
      *
+ * + * @see #uploadFile(Consumer) + * @see #upload(UploadRequest) */ - default Upload upload(UploadRequest uploadRequest) { + default FileUpload uploadFile(UploadFileRequest uploadFileRequest) { throw new UnsupportedOperationException(); } /** - * Upload a file to S3. - * - *

- * This is a convenience method that creates an instance of the {@link UploadRequest} builder avoiding the - * need to create one manually via {@link UploadRequest#builder()}. + * This is a convenience method that creates an instance of the {@link UploadFileRequest} builder, avoiding the need to create + * one manually via {@link UploadFileRequest#builder()}. * + * @see #uploadFile(UploadFileRequest) + */ + default FileUpload uploadFile(Consumer request) { + return uploadFile(UploadFileRequest.builder().applyMutation(request).build()); + } + + /** + * Upload an {@link AsyncRequestBody} to S3. *

* Usage Example: * 

      * {@code
-     * Upload upload =
-     *       transferManager.upload(b -> b.putObjectRequest(req -> req.bucket("bucket")
-     *                                                                .key("key"))
-     *                                    .source(Paths.get("myFile.txt")));
+     * Upload upload = tm.upload(UploadRequest.builder()
+     *                                        .putObjectRequest(b -> b.bucket("bucket"").key("key""))
+     *                                        .requestBody(AsyncRequestBody.fromString("Hello world"))
+     *                                        .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
+     *                                        .build());
      * // Wait for the transfer to complete
      * upload.completionFuture().join();
      * }
      *
+ * See the static factory methods available in {@link AsyncRequestBody} for other use cases. + * + * @param uploadRequest the upload request, containing a {@link PutObjectRequest} and {@link AsyncRequestBody} + * @return An {@link Upload} that can be used to track the ongoing transfer + * @see #upload(Consumer) + * @see #uploadFile(UploadFileRequest) + */ + default Upload upload(UploadRequest uploadRequest) { + throw new UnsupportedOperationException(); + } + + /** + * This is a convenience method that creates an instance of the {@link UploadRequest} builder, avoiding the need to create one + * manually via {@link UploadRequest#builder()}. + * + * @see #upload(UploadRequest) */ default Upload upload(Consumer request) { return upload(UploadRequest.builder().applyMutation(request).build()); @@ -175,7 +231,7 @@ default Upload upload(Consumer request) { * The returned {@link CompletableFuture} only completes exceptionally if the request cannot be attempted as a whole (the * source directory provided does not exist for example). The future completes successfully for partial successful * requests, i.e., there might be failed uploads in the successfully completed response. As a result, - * you should check for errors in the response via {@link CompletedUploadDirectory#failedUploads()} + * you should check for errors in the response via {@link CompletedDirectoryUpload#failedTransfers()} * even when the future completes successfully. * *

@@ -185,17 +241,17 @@ default Upload upload(Consumer request) { * Usage Example: * 

      * {@code
-     * UploadDirectoryTransfer uploadDirectory =
+     * DirectoryUpload directoryUpload =
      *       transferManager.uploadDirectory(UploadDirectoryRequest.builder()
      *                                                             .sourceDirectory(Paths.get("."))
      *                                                             .bucket("bucket")
      *                                                             .prefix("prefix")
      *                                                             .build());
      * // Wait for the transfer to complete
-     * CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
+     * CompletedDirectoryUpload completedDirectoryUpload = directoryUpload.completionFuture().join();
      *
      * // Print out the failed uploads
-     * completedUploadDirectory.failedUploads().forEach(System.out::println);
+     * completedDirectoryUpload.failedTransfers().forEach(System.out::println);
      *
      * }
      *
@@ -204,51 +260,17 @@ default Upload upload(Consumer request) { * @see #uploadDirectory(Consumer) * @see UploadDirectoryOverrideConfiguration */ - default UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { + default DirectoryUpload uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { throw new UnsupportedOperationException(); } /** - * Upload all files under the given directory to the provided S3 bucket. The key name transformation depends on the optional - * prefix and delimiter provided in the {@link UploadDirectoryRequest}. By default, all subdirectories will be uploaded - * recursively, and symbolic links are not followed automatically. This behavior can be configured in - * {@link UploadDirectoryOverrideConfiguration} - * at request level via {@link UploadDirectoryRequest.Builder#overrideConfiguration(UploadDirectoryOverrideConfiguration)} or - * client level via {@link S3TransferManager.Builder#transferConfiguration(S3TransferManagerOverrideConfiguration)} Note - * that request-level configuration takes precedence over client-level configuration. - * - *

- * The returned {@link CompletableFuture} only completes exceptionally if the request cannot be attempted as a whole (the - * source directory provided does not exist for example). The future completes successfully for partial successful - * requests, i.e., there might be failed uploads in the successfully completed response. As a result, - * you should check for errors in the response via {@link CompletedUploadDirectory#failedUploads()} - * even when the future completes successfully. - * - *

- * The current user must have read access to all directories and files - * - *

- * This is a convenience method that creates an instance of the {@link UploadDirectoryRequest} builder avoiding the - * need to create one manually via {@link UploadDirectoryRequest#builder()}. - * - *

- * Usage Example: - * 

-     * {@code
-     * UploadDirectoryTransfer uploadDirectory =
-     *       transferManager.uploadDirectory(b -> b.sourceDirectory(Paths.get("."))
-     *                                             .bucket("key")
-     *                                             .prefix("prefix"));
-     * // Print out the failed uploads
-     * completedUploadDirectory.failedUploads().forEach(System.out::println);
+     * This is a convenience method that creates an instance of the {@link UploadDirectoryRequest} builder, avoiding the need to
+     * create one manually via {@link UploadDirectoryRequest#builder()}.
      *
-     * }
-     *
- * @param requestBuilder the upload directory request builder * @see #uploadDirectory(UploadDirectoryRequest) - * @see UploadDirectoryOverrideConfiguration */ - default UploadDirectoryTransfer uploadDirectory(Consumer requestBuilder) { + default DirectoryUpload uploadDirectory(Consumer requestBuilder) { Validate.paramNotNull(requestBuilder, "requestBuilder"); return uploadDirectory(UploadDirectoryRequest.builder().applyMutation(requestBuilder).build()); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java index eb91799f363a..3bbd57676fed 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java @@ -21,6 +21,9 @@ /** * Represents the upload or download of one or more objects to or from S3. + * + * @see ObjectTransfer + * @see DirectoryTransfer */ @SdkPublicApi @SdkPreviewApi diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferDirectoryRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferDirectoryRequest.java new file mode 100644 index 000000000000..b8633723f2d9 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferDirectoryRequest.java @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Interface for all transfer directory requests. + * + * @see UploadDirectoryRequest + */ +@SdkPublicApi +@SdkPreviewApi +public interface TransferDirectoryRequest extends TransferRequest { +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferObjectRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferObjectRequest.java new file mode 100644 index 000000000000..f444c8da7c21 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferObjectRequest.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Interface for all single object transfer requests. + * + * @see UploadFileRequest + * @see DownloadFileRequest + * @see UploadRequest + * @see DownloadRequest + */ +@SdkPublicApi +@SdkPreviewApi +public interface TransferObjectRequest extends TransferRequest { + + Optional overrideConfiguration(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java index 21317003e91b..e4d38dd213c4 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java @@ -19,14 +19,12 @@ import software.amazon.awssdk.annotations.SdkPublicApi; /** - * Interface for all transfer requests. + * The parent interface for all transfer requests. + * + * @see TransferObjectRequest + * @see TransferDirectoryRequest */ @SdkPublicApi @SdkPreviewApi public interface TransferRequest { - - interface Builder { - - TypeToBuildT build(); - } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequestOverrideConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequestOverrideConfiguration.java index bbbdacaaeeae..4d22146ff65d 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequestOverrideConfiguration.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequestOverrideConfiguration.java @@ -28,8 +28,8 @@ import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** - * Configuration options for {@link UploadRequest} and {@link DownloadRequest}. All values are optional, and not specifying them - * will use the SDK default values. + * Configuration options for {@link UploadFileRequest} and {@link DownloadFileRequest}. All values are optional, and not + * specifying them will use the SDK default values. * *

Use {@link #builder()} to create a set of options. */ diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java index 1c7b56c17d2f..e59cc1b60702 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java @@ -18,19 +18,13 @@ import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.transfer.s3.progress.TransferProgress; /** * An upload transfer of a single object to S3. */ @SdkPublicApi @SdkPreviewApi -public interface Upload extends Transfer { +public interface Upload extends ObjectTransfer { @Override CompletableFuture completionFuture(); - - /** - * The stateful {@link TransferProgress} associated with this transfer. - */ - TransferProgress progress(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java index fcf9cbb1c46c..1ee86a4e1517 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java @@ -22,6 +22,7 @@ import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -33,8 +34,8 @@ */ @SdkPublicApi @SdkPreviewApi -public final class UploadDirectoryRequest implements TransferRequest, ToCopyableBuilder { +public final class UploadDirectoryRequest + implements TransferDirectoryRequest, ToCopyableBuilder { private final Path sourceDirectory; private final String bucket; @@ -118,31 +119,42 @@ public boolean equals(Object o) { UploadDirectoryRequest that = (UploadDirectoryRequest) o; - if (!sourceDirectory.equals(that.sourceDirectory)) { + if (!Objects.equals(sourceDirectory, that.sourceDirectory)) { return false; } - if (!bucket.equals(that.bucket)) { + if (!Objects.equals(bucket, that.bucket)) { return false; } if (!Objects.equals(prefix, that.prefix)) { return false; } - if (!Objects.equals(delimiter, that.delimiter)) { + if (!Objects.equals(overrideConfiguration, that.overrideConfiguration)) { return false; } - return Objects.equals(overrideConfiguration, that.overrideConfiguration); + return Objects.equals(delimiter, that.delimiter); } @Override public int hashCode() { - int result = sourceDirectory.hashCode(); - result = 31 * result + bucket.hashCode(); + int result = sourceDirectory != null ? sourceDirectory.hashCode() : 0; + result = 31 * result + (bucket != null ? bucket.hashCode() : 0); result = 31 * result + (prefix != null ? prefix.hashCode() : 0); - result = 31 * result + (delimiter != null ? delimiter.hashCode() : 0); result = 31 * result + (overrideConfiguration != null ? overrideConfiguration.hashCode() : 0); + result = 31 * result + (delimiter != null ? delimiter.hashCode() : 0); return result; } + @Override + public String toString() { + return ToString.builder("UploadDirectoryRequest") + .add("sourceDirectory", sourceDirectory) + .add("bucket", bucket) + .add("prefix", prefix) + .add("overrideConfiguration", overrideConfiguration) + .add("delimiter", delimiter) + .build(); + } + public interface Builder extends CopyableBuilder { /** diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java deleted file mode 100644 index 39ef2d7b6736..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3; - -import java.util.concurrent.CompletableFuture; -import software.amazon.awssdk.annotations.SdkPreviewApi; -import software.amazon.awssdk.annotations.SdkPublicApi; - -@SdkPublicApi -@SdkPreviewApi -public final class UploadDirectoryTransfer implements Transfer { - private final CompletableFuture completionFuture; - - private UploadDirectoryTransfer(DefaultBuilder builder) { - this.completionFuture = builder.completionFuture; - } - - @Override - public CompletableFuture completionFuture() { - return completionFuture; - } - - public static Builder builder() { - return new DefaultBuilder(); - } - - public static Class serializableBuilderClass() { - return DefaultBuilder.class; - } - - public interface Builder { - - /** - * Specifies the future that will be completed when this transfer is complete. - * - * @param completionFuture the future that will be completed when this transfer is complete. - * @return This builder for method chaining. - */ - Builder completionFuture(CompletableFuture completionFuture); - - /** - * Builds a {@link UploadDirectoryTransfer} based on the properties supplied to this builder - * - * @return An initialized {@link UploadDirectoryTransfer} - */ - UploadDirectoryTransfer build(); - } - - private static final class DefaultBuilder implements Builder { - private CompletableFuture completionFuture; - - private DefaultBuilder() { - } - - @Override - public DefaultBuilder completionFuture(CompletableFuture completionFuture) { - this.completionFuture = completionFuture; - return this; - } - - public void setCompletionFuture(CompletableFuture completionFuture) { - completionFuture(completionFuture); - } - - @Override - public UploadDirectoryTransfer build() { - return new UploadDirectoryTransfer(this); - } - } -} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadFileRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadFileRequest.java new file mode 100644 index 000000000000..9729fdf8bef1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadFileRequest.java @@ -0,0 +1,276 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static software.amazon.awssdk.utils.Validate.paramNotNull; + +import java.io.File; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Upload an object to S3 using {@link S3TransferManager}. + * @see S3TransferManager#uploadFile(UploadFileRequest) + */ +@SdkPublicApi +@SdkPreviewApi +public final class UploadFileRequest + implements TransferObjectRequest, + ToCopyableBuilder { + + private final PutObjectRequest putObjectRequest; + private final Path source; + private final TransferRequestOverrideConfiguration configuration; + + private UploadFileRequest(DefaultBuilder builder) { + this.putObjectRequest = paramNotNull(builder.putObjectRequest, "putObjectRequest"); + this.source = paramNotNull(builder.source, "source"); + this.configuration = builder.configuration; + } + + /** + * @return The {@link PutObjectRequest} request that should be used for the upload + */ + public PutObjectRequest putObjectRequest() { + return putObjectRequest; + } + + /** + * The {@link Path} containing data to send to the service. + * + * @return the request body + */ + public Path source() { + return source; + } + + /** + * @return the optional override configuration + * @see Builder#overrideConfiguration(TransferRequestOverrideConfiguration) + */ + @Override + public Optional overrideConfiguration() { + return Optional.ofNullable(configuration); + } + + /** + * Create a builder that can be used to create a {@link UploadFileRequest}. + * + * @see S3TransferManager#uploadFile(UploadFileRequest) + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + UploadFileRequest that = (UploadFileRequest) o; + + if (!Objects.equals(putObjectRequest, that.putObjectRequest)) { + return false; + } + if (!Objects.equals(source, that.source)) { + return false; + } + return Objects.equals(configuration, that.configuration); + } + + @Override + public int hashCode() { + int result = putObjectRequest != null ? putObjectRequest.hashCode() : 0; + result = 31 * result + (source != null ? source.hashCode() : 0); + result = 31 * result + (configuration != null ? configuration.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("UploadFileRequest") + .add("putObjectRequest", putObjectRequest) + .add("source", source) + .add("configuration", configuration) + .build(); + } + + /** + * A builder for a {@link UploadFileRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends CopyableBuilder { + + /** + * The {@link Path} to file containing data to send to the service. File will be read entirely and may be read + * multiple times in the event of a retry. If the file does not exist or the current user does not have + * access to read it then an exception will be thrown. + * + * @param source the source path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder source(Path source); + + /** + * The file containing data to send to the service. File will be read entirely and may be read + * multiple times in the event of a retry. If the file does not exist or the current user does not have + * access to read it then an exception will be thrown. + * + * @param source the source path + * @return Returns a reference to this object so that method calls can be chained together. + */ + default Builder source(File source) { + Validate.paramNotNull(source, "source"); + return this.source(source.toPath()); + } + + /** + * Configure the {@link PutObjectRequest} that should be used for the upload + * + * @param putObjectRequest the putObjectRequest + * @return Returns a reference to this object so that method calls can be chained together. + * @see #putObjectRequest(Consumer) + */ + Builder putObjectRequest(PutObjectRequest putObjectRequest); + + /** + * Configure the {@link PutObjectRequest} that should be used for the upload + * + *

+ * This is a convenience method that creates an instance of the {@link PutObjectRequest} builder avoiding the + * need to create one manually via {@link PutObjectRequest#builder()}. + * + * @param putObjectRequestBuilder the putObjectRequest consumer builder + * @return Returns a reference to this object so that method calls can be chained together. + * @see #putObjectRequest(PutObjectRequest) + */ + default Builder putObjectRequest(Consumer putObjectRequestBuilder) { + return putObjectRequest(PutObjectRequest.builder() + .applyMutation(putObjectRequestBuilder) + .build()); + } + + /** + * Add an optional request override configuration. + * + * @param configuration The override configuration. + * @return This builder for method chaining. + */ + Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration); + + /** + * Similar to {@link #overrideConfiguration(TransferRequestOverrideConfiguration)}, but takes a lambda to configure a new + * {@link TransferRequestOverrideConfiguration.Builder}. This removes the need to call {@link + * TransferRequestOverrideConfiguration#builder()} and {@link TransferRequestOverrideConfiguration.Builder#build()}. + * + * @param configurationBuilder the upload configuration + * @return this builder for method chaining. + * @see #overrideConfiguration(TransferRequestOverrideConfiguration) + */ + default Builder overrideConfiguration(Consumer configurationBuilder) { + Validate.paramNotNull(configurationBuilder, "configurationBuilder"); + return overrideConfiguration(TransferRequestOverrideConfiguration.builder() + .applyMutation(configurationBuilder) + .build()); + } + } + + private static class DefaultBuilder implements Builder { + private PutObjectRequest putObjectRequest; + private Path source; + private TransferRequestOverrideConfiguration configuration; + + private DefaultBuilder() { + } + + private DefaultBuilder(UploadFileRequest uploadFileRequest) { + this.source = uploadFileRequest.source; + this.putObjectRequest = uploadFileRequest.putObjectRequest; + this.configuration = uploadFileRequest.configuration; + } + + @Override + public Builder source(Path source) { + this.source = Validate.paramNotNull(source, "source"); + return this; + } + + public Path getSource() { + return source; + } + + public void setSource(Path source) { + source(source); + } + + @Override + public Builder putObjectRequest(PutObjectRequest putObjectRequest) { + this.putObjectRequest = putObjectRequest; + return this; + } + + public PutObjectRequest getPutObjectRequest() { + return putObjectRequest; + } + + public void setPutObjectRequest(PutObjectRequest putObjectRequest) { + putObjectRequest(putObjectRequest); + } + + @Override + public Builder overrideConfiguration(TransferRequestOverrideConfiguration configuration) { + this.configuration = configuration; + return this; + } + + public void setOverrideConfiguration(TransferRequestOverrideConfiguration configuration) { + overrideConfiguration(configuration); + } + + public TransferRequestOverrideConfiguration getOverrideConfiguration() { + return configuration; + } + + @Override + public UploadFileRequest build() { + return new UploadFileRequest(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java index c77951eca233..99e31a1085d5 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java @@ -17,7 +17,6 @@ import static software.amazon.awssdk.utils.Validate.paramNotNull; -import java.io.File; import java.nio.file.Path; import java.util.Objects; import java.util.Optional; @@ -25,6 +24,7 @@ import software.amazon.awssdk.annotations.NotThreadSafe; import software.amazon.awssdk.annotations.SdkPreviewApi; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; @@ -37,15 +37,18 @@ */ @SdkPublicApi @SdkPreviewApi -public final class UploadRequest implements TransferRequest, ToCopyableBuilder { +public final class UploadRequest + implements TransferObjectRequest, + ToCopyableBuilder { + private final PutObjectRequest putObjectRequest; - private final Path source; - private final TransferRequestOverrideConfiguration overrideConfiguration; + private final AsyncRequestBody requestBody; + private final TransferRequestOverrideConfiguration configuration; - private UploadRequest(BuilderImpl builder) { + private UploadRequest(DefaultBuilder builder) { this.putObjectRequest = paramNotNull(builder.putObjectRequest, "putObjectRequest"); - this.source = paramNotNull(builder.source, "source"); - this.overrideConfiguration = builder.configuration; + this.requestBody = paramNotNull(builder.requestBody, "requestBody"); + this.configuration = builder.configuration; } /** @@ -56,20 +59,21 @@ public PutObjectRequest putObjectRequest() { } /** - * The {@link Path} to file containing data to send to the service. + * The {@link AsyncRequestBody} containing data to send to the service. * - * @return the source path + * @return the request body */ - public Path source() { - return source; + public AsyncRequestBody requestBody() { + return requestBody; } /** * @return the optional override configuration * @see Builder#overrideConfiguration(TransferRequestOverrideConfiguration) */ + @Override public Optional overrideConfiguration() { - return Optional.ofNullable(overrideConfiguration); + return Optional.ofNullable(configuration); } /** @@ -78,25 +82,16 @@ public Optional overrideConfiguration() { * @see S3TransferManager#upload(UploadRequest) */ public static Builder builder() { - return new BuilderImpl(); + return new DefaultBuilder(); } public static Class serializableBuilderClass() { - return BuilderImpl.class; + return DefaultBuilder.class; } - + @Override public Builder toBuilder() { - return new BuilderImpl(); - } - - @Override - public String toString() { - return ToString.builder("UploadRequest") - .add("putObjectRequest", putObjectRequest) - .add("source", source) - .add("overrideConfiguration", overrideConfiguration) - .build(); + return new DefaultBuilder(this); } @Override @@ -113,49 +108,47 @@ public boolean equals(Object o) { if (!Objects.equals(putObjectRequest, that.putObjectRequest)) { return false; } - if (!Objects.equals(source, that.source)) { + if (!Objects.equals(requestBody, that.requestBody)) { return false; } - return Objects.equals(overrideConfiguration, that.overrideConfiguration); + return Objects.equals(configuration, that.configuration); } @Override public int hashCode() { int result = putObjectRequest != null ? putObjectRequest.hashCode() : 0; - result = 31 * result + (source != null ? source.hashCode() : 0); - result = 31 * result + (overrideConfiguration != null ? overrideConfiguration.hashCode() : 0); + result = 31 * result + (requestBody != null ? requestBody.hashCode() : 0); + result = 31 * result + (configuration != null ? configuration.hashCode() : 0); return result; } + @Override + public String toString() { + return ToString.builder("UploadRequest") + .add("putObjectRequest", putObjectRequest) + .add("requestBody", requestBody) + .add("configuration", configuration) + .build(); + } + /** * A builder for a {@link UploadRequest}, created with {@link #builder()} */ @SdkPublicApi @NotThreadSafe - public interface Builder extends TransferRequest.Builder, CopyableBuilder { + public interface Builder extends CopyableBuilder { /** - * The {@link Path} to file containing data to send to the service. File will be read entirely and may be read - * multiple times in the event of a retry. If the file does not exist or the current user does not have - * access to read it then an exception will be thrown. + * The {@link AsyncRequestBody} containing the data to send to the service. Request bodies may be declared using one of + * the static factory methods in the {@link AsyncRequestBody} class, or in the case of file-based requests, with the + * builder method: {@link #source(Path)}. * - * @param source the source path + * @param requestBody the request body * @return Returns a reference to this object so that method calls can be chained together. + * @see AsyncRequestBody + * @see #source(Path) */ - Builder source(Path source); - - /** - * The file containing data to send to the service. File will be read entirely and may be read - * multiple times in the event of a retry. If the file does not exist or the current user does not have - * access to read it then an exception will be thrown. - * - * @param source the source path - * @return Returns a reference to this object so that method calls can be chained together. - */ - default Builder source(File source) { - Validate.paramNotNull(source, "source"); - return this.source(source.toPath()); - } + Builder requestBody(AsyncRequestBody requestBody); /** * Configure the {@link PutObjectRequest} that should be used for the upload @@ -214,23 +207,32 @@ default Builder overrideConfiguration(Consumer completionFuture; + + DefaultDirectoryUpload(CompletableFuture completionFuture) { + this.completionFuture = completionFuture; + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultDirectoryUpload that = (DefaultDirectoryUpload) o; + + return Objects.equals(completionFuture, that.completionFuture); + } + + @Override + public int hashCode() { + return completionFuture != null ? completionFuture.hashCode() : 0; + } + + @Override + public String toString() { + return ToString.builder("DefaultDirectoryUpload") + .add("completionFuture", completionFuture) + .build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java index 0c5611759f0a..78f43ecba4ee 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java @@ -15,24 +15,27 @@ package software.amazon.awssdk.transfer.s3.internal; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.transfer.s3.CompletedDownload; import software.amazon.awssdk.transfer.s3.Download; import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; @SdkInternalApi -public final class DefaultDownload implements Download { - private final CompletableFuture completionFuture; +public final class DefaultDownload implements Download { + + private final CompletableFuture> completionFuture; private final TransferProgress progress; - public DefaultDownload(CompletableFuture completionFuture, TransferProgress progress) { + DefaultDownload(CompletableFuture> completionFuture, TransferProgress progress) { this.completionFuture = completionFuture; this.progress = progress; } @Override - public CompletableFuture completionFuture() { + public CompletableFuture> completionFuture() { return completionFuture; } @@ -40,4 +43,36 @@ public CompletableFuture completionFuture() { public TransferProgress progress() { return progress; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultDownload that = (DefaultDownload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultDownload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownload.java new file mode 100644 index 000000000000..b4cba4a7bf93 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownload.java @@ -0,0 +1,80 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.FileDownload; +import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; + +@SdkInternalApi +public final class DefaultFileDownload implements FileDownload { + + private final CompletableFuture completionFuture; + private final TransferProgress progress; + + DefaultFileDownload(CompletableFuture completionFuture, + TransferProgress progress) { + this.completionFuture = completionFuture; + this.progress = progress; + } + + @Override + public TransferProgress progress() { + return progress; + } + + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultFileDownload that = (DefaultFileDownload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultFileDownload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUpload.java new file mode 100644 index 000000000000..de5e13853475 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUpload.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.FileUpload; +import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; + +@SdkInternalApi +public final class DefaultFileUpload implements FileUpload { + + private final CompletableFuture completionFuture; + private final TransferProgress progress; + + DefaultFileUpload(CompletableFuture completionFuture, TransferProgress progress) { + this.completionFuture = completionFuture; + this.progress = progress; + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public TransferProgress progress() { + return progress; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultFileUpload that = (DefaultFileUpload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultFileUpload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java index dd9e83dfe30c..a0c08ff6951d 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java @@ -29,15 +29,21 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.transfer.s3.CompletedDownload; +import software.amazon.awssdk.transfer.s3.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; import software.amazon.awssdk.transfer.s3.CompletedUpload; +import software.amazon.awssdk.transfer.s3.DirectoryUpload; import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.transfer.s3.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.FileDownload; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.S3ClientConfiguration; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.S3TransferManagerOverrideConfiguration; import software.amazon.awssdk.transfer.s3.Upload; import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest; -import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; import software.amazon.awssdk.transfer.s3.UploadRequest; import software.amazon.awssdk.transfer.s3.internal.progress.TransferProgressUpdater; import software.amazon.awssdk.utils.CompletableFutureUtils; @@ -52,7 +58,7 @@ public final class DefaultS3TransferManager implements S3TransferManager { public DefaultS3TransferManager(DefaultBuilder tmBuilder) { transferConfiguration = resolveTransferManagerConfiguration(tmBuilder); s3CrtAsyncClient = initializeS3CrtClient(tmBuilder); - uploadDirectoryManager = new UploadDirectoryHelper(transferConfiguration, this::upload); + uploadDirectoryManager = new UploadDirectoryHelper(transferConfiguration, this::uploadFile); } @SdkTestInternalApi @@ -93,10 +99,10 @@ private S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder) { public Upload upload(UploadRequest uploadRequest) { Validate.paramNotNull(uploadRequest, "uploadRequest"); - AsyncRequestBody requestBody = requestBodyFor(uploadRequest); - + AsyncRequestBody requestBody = uploadRequest.requestBody(); + CompletableFuture uploadFuture = new CompletableFuture<>(); - + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(uploadRequest, requestBody); progressUpdater.transferInitiated(); requestBody = progressUpdater.wrapRequestBody(requestBody); @@ -107,22 +113,56 @@ public Upload upload(UploadRequest uploadRequest) { CompletableFuture putObjFuture = s3CrtAsyncClient.putObject(uploadRequest.putObjectRequest(), requestBody); - + // Forward upload cancellation to CRT future CompletableFutureUtils.forwardExceptionTo(uploadFuture, putObjFuture); - CompletableFutureUtils.forwardTransformedResultTo(putObjFuture, uploadFuture, r -> CompletedUpload.builder() - .response(r) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(putObjFuture, uploadFuture, + r -> CompletedUpload.builder() + .response(r) + .build()); } catch (Throwable throwable) { uploadFuture.completeExceptionally(throwable); } return new DefaultUpload(uploadFuture, progressUpdater.progress()); } + + @Override + public FileUpload uploadFile(UploadFileRequest uploadFileRequest) { + Validate.paramNotNull(uploadFileRequest, "uploadFileRequest"); + + AsyncRequestBody requestBody = AsyncRequestBody.fromFile(uploadFileRequest.source()); + + CompletableFuture uploadFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(uploadFileRequest, requestBody); + progressUpdater.transferInitiated(); + requestBody = progressUpdater.wrapRequestBody(requestBody); + progressUpdater.registerCompletion(uploadFuture); + + try { + assertNotUnsupportedArn(uploadFileRequest.putObjectRequest().bucket(), "upload"); + + CompletableFuture putObjFuture = + s3CrtAsyncClient.putObject(uploadFileRequest.putObjectRequest(), requestBody); + + // Forward upload cancellation to CRT future + CompletableFutureUtils.forwardExceptionTo(uploadFuture, putObjFuture); + + CompletableFutureUtils.forwardTransformedResultTo(putObjFuture, uploadFuture, + r -> CompletedFileUpload.builder() + .response(r) + .build()); + } catch (Throwable throwable) { + uploadFuture.completeExceptionally(throwable); + } + + return new DefaultFileUpload(uploadFuture, progressUpdater.progress()); + } @Override - public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { + public DirectoryUpload uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { Validate.paramNotNull(uploadDirectoryRequest, "uploadDirectoryRequest"); try { @@ -130,20 +170,54 @@ public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDire return uploadDirectoryManager.uploadDirectory(uploadDirectoryRequest); } catch (Throwable throwable) { - return UploadDirectoryTransfer.builder().completionFuture(CompletableFutureUtils.failedFuture(throwable)).build(); + return new DefaultDirectoryUpload(CompletableFutureUtils.failedFuture(throwable)); } } @Override - public Download download(DownloadRequest downloadRequest) { + public Download download(DownloadRequest downloadRequest) { Validate.paramNotNull(downloadRequest, "downloadRequest"); + AsyncResponseTransformer responseTransformer = + downloadRequest.responseTransformer(); + + CompletableFuture> downloadFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(downloadRequest, null); + progressUpdater.transferInitiated(); + responseTransformer = progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(downloadFuture); + + try { + assertNotUnsupportedArn(downloadRequest.getObjectRequest().bucket(), "download"); + + CompletableFuture getObjectFuture = + s3CrtAsyncClient.getObject(downloadRequest.getObjectRequest(), responseTransformer); + + // Forward download cancellation to CRT future + CompletableFutureUtils.forwardExceptionTo(downloadFuture, getObjectFuture); + + CompletableFutureUtils.forwardTransformedResultTo(getObjectFuture, downloadFuture, + r -> CompletedDownload.builder() + .result(r) + .build()); + } catch (Throwable throwable) { + downloadFuture.completeExceptionally(throwable); + } + + return new DefaultDownload<>(downloadFuture, progressUpdater.progress()); + } + + @Override + public FileDownload downloadFile(DownloadFileRequest downloadRequest) { + Validate.paramNotNull(downloadRequest, "downloadFileRequest"); + AsyncResponseTransformer responseTransformer = AsyncResponseTransformer.toFile(downloadRequest.destination()); - CompletableFuture downloadFuture = new CompletableFuture<>(); + CompletableFuture downloadFuture = new CompletableFuture<>(); - TransferProgressUpdater progressUpdater = new TransferProgressUpdater(downloadRequest); + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(downloadRequest, null); progressUpdater.transferInitiated(); responseTransformer = progressUpdater.wrapResponseTransformer(responseTransformer); progressUpdater.registerCompletion(downloadFuture); @@ -157,14 +231,15 @@ public Download download(DownloadRequest downloadRequest) { // Forward download cancellation to CRT future CompletableFutureUtils.forwardExceptionTo(downloadFuture, getObjectFuture); - CompletableFutureUtils.forwardTransformedResultTo(getObjectFuture, downloadFuture, r -> CompletedDownload.builder() - .response(r) - .build()); + CompletableFutureUtils.forwardTransformedResultTo(getObjectFuture, downloadFuture, + r -> CompletedFileDownload.builder() + .response(r) + .build()); } catch (Throwable throwable) { downloadFuture.completeExceptionally(throwable); } - return new DefaultDownload(downloadFuture, progressUpdater.progress()); + return new DefaultFileDownload(downloadFuture, progressUpdater.progress()); } @Override @@ -210,10 +285,6 @@ private static boolean isMrapArn(Arn arn) { return !s3EndpointResource.region().isPresent(); } - private AsyncRequestBody requestBodyFor(UploadRequest uploadRequest) { - return AsyncRequestBody.fromFile(uploadRequest.source()); - } - private static class DefaultBuilder implements S3TransferManager.Builder { private S3ClientConfiguration s3ClientConfiguration = S3ClientConfiguration.builder().build(); private S3TransferManagerOverrideConfiguration transferManagerConfiguration = diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java index f1423adf0b37..12681d0da591 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java @@ -15,18 +15,21 @@ package software.amazon.awssdk.transfer.s3.internal; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.transfer.s3.CompletedUpload; import software.amazon.awssdk.transfer.s3.Upload; import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; @SdkInternalApi public final class DefaultUpload implements Upload { + private final CompletableFuture completionFuture; private final TransferProgress progress; - public DefaultUpload(CompletableFuture completionFuture, TransferProgress progress) { + DefaultUpload(CompletableFuture completionFuture, TransferProgress progress) { this.completionFuture = completionFuture; this.progress = progress; } @@ -40,4 +43,36 @@ public CompletableFuture completionFuture() { public TransferProgress progress() { return progress; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultUpload that = (DefaultUpload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultUpload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java index 64d64bddacc7..66a5c44d0131 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java @@ -35,14 +35,14 @@ import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory; +import software.amazon.awssdk.transfer.s3.CompletedDirectoryUpload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.DirectoryUpload; import software.amazon.awssdk.transfer.s3.FailedFileUpload; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.S3TransferManager; -import software.amazon.awssdk.transfer.s3.Upload; import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest; -import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.StringUtils; @@ -57,11 +57,11 @@ public class UploadDirectoryHelper { private static final Logger log = Logger.loggerFor(S3TransferManager.class); private final TransferManagerConfiguration transferConfiguration; - private final Function uploadFunction; + private final Function uploadFunction; private final FileSystem fileSystem; public UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration, - Function uploadFunction) { + Function uploadFunction) { this.transferConfiguration = transferConfiguration; this.uploadFunction = uploadFunction; @@ -70,7 +70,7 @@ public UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration, @SdkTestInternalApi UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration, - Function uploadFunction, + Function uploadFunction, FileSystem fileSystem) { this.transferConfiguration = transferConfiguration; @@ -78,9 +78,9 @@ public UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration, this.fileSystem = fileSystem; } - public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { + public DirectoryUpload uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) { - CompletableFuture returnFuture = new CompletableFuture<>(); + CompletableFuture returnFuture = new CompletableFuture<>(); // offload the execution to the transfer manager executor CompletableFuture.runAsync(() -> doUploadDirectory(returnFuture, uploadDirectoryRequest), @@ -91,23 +91,23 @@ public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDire } }); - return UploadDirectoryTransfer.builder().completionFuture(returnFuture).build(); + return new DefaultDirectoryUpload(returnFuture); } - private void doUploadDirectory(CompletableFuture returnFuture, + private void doUploadDirectory(CompletableFuture returnFuture, UploadDirectoryRequest uploadDirectoryRequest) { Path directory = uploadDirectoryRequest.sourceDirectory(); validateDirectory(uploadDirectoryRequest); - Collection failedUploads = new ConcurrentLinkedQueue<>(); - List> futures; + Collection failedFileUploads = new ConcurrentLinkedQueue<>(); + List> futures; try (Stream entries = listFiles(directory, uploadDirectoryRequest)) { futures = entries.map(path -> { - CompletableFuture future = uploadSingleFile(uploadDirectoryRequest, - failedUploads, path); + CompletableFuture future = uploadSingleFile(uploadDirectoryRequest, + failedFileUploads, path); // Forward cancellation of the return future to all individual futures. CompletableFutureUtils.forwardExceptionTo(returnFuture, future); @@ -116,8 +116,8 @@ private void doUploadDirectory(CompletableFuture retur } CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .whenComplete((r, t) -> returnFuture.complete(CompletedUploadDirectory.builder() - .failedUploads(failedUploads) + .whenComplete((r, t) -> returnFuture.complete(CompletedDirectoryUpload.builder() + .failedTransfers(failedFileUploads) .build())); } @@ -135,19 +135,19 @@ private void validateDirectory(UploadDirectoryRequest uploadDirectoryRequest) { } } - private CompletableFuture uploadSingleFile(UploadDirectoryRequest uploadDirectoryRequest, - Collection failedUploads, - Path path) { + private CompletableFuture uploadSingleFile(UploadDirectoryRequest uploadDirectoryRequest, + Collection failedFileUploads, + Path path) { int nameCount = uploadDirectoryRequest.sourceDirectory().getNameCount(); - UploadRequest uploadRequest = constructUploadRequest(uploadDirectoryRequest, nameCount, path); - log.debug(() -> String.format("Sending upload request (%s) for path (%s)", uploadRequest, path)); - CompletableFuture future = uploadFunction.apply(uploadRequest).completionFuture(); + UploadFileRequest uploadFileRequest = constructUploadRequest(uploadDirectoryRequest, nameCount, path); + log.debug(() -> String.format("Sending upload request (%s) for path (%s)", uploadFileRequest, path)); + CompletableFuture future = uploadFunction.apply(uploadFileRequest).completionFuture(); future.whenComplete((r, t) -> { if (t != null) { - failedUploads.add(FailedFileUpload.builder() - .exception(t) - .request(uploadRequest) - .build()); + failedFileUploads.add(FailedFileUpload.builder() + .exception(t) + .request(uploadFileRequest) + .build()); } }); return future; @@ -212,8 +212,8 @@ private String getRelativePathName(int directoryNameCount, Path path, String del return relativePathName.replace(separator, delimiter); } - private UploadRequest constructUploadRequest(UploadDirectoryRequest uploadDirectoryRequest, int directoryNameCount, - Path path) { + private UploadFileRequest constructUploadRequest(UploadDirectoryRequest uploadDirectoryRequest, int directoryNameCount, + Path path) { String delimiter = uploadDirectoryRequest.delimiter() .filter(s -> !s.isEmpty()) @@ -230,9 +230,9 @@ private UploadRequest constructUploadRequest(UploadDirectoryRequest uploadDirect .bucket(uploadDirectoryRequest.bucket()) .key(key) .build(); - return UploadRequest.builder() - .source(path) - .putObjectRequest(putObjectRequest) - .build(); + return UploadFileRequest.builder() + .source(path) + .putObjectRequest(putObjectRequest) + .build(); } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerContext.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerContext.java index 4437bb601ae4..19f409db07d5 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerContext.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerContext.java @@ -17,8 +17,8 @@ import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkProtectedApi; -import software.amazon.awssdk.transfer.s3.CompletedTransfer; -import software.amazon.awssdk.transfer.s3.TransferRequest; +import software.amazon.awssdk.transfer.s3.CompletedObjectTransfer; +import software.amazon.awssdk.transfer.s3.TransferObjectRequest; import software.amazon.awssdk.transfer.s3.progress.TransferListener; import software.amazon.awssdk.transfer.s3.progress.TransferProgressSnapshot; import software.amazon.awssdk.utils.ToString; @@ -26,7 +26,7 @@ import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** - * An SDK-internal implementation of {@link TransferComplete} and its parent interfaces. + * An SDK-internal implementation of {@link TransferListener.Context.TransferComplete} and its parent interfaces. * * @see TransferListenerFailedContext */ @@ -36,9 +36,9 @@ public final class TransferListenerContext implements TransferListener.Context.TransferComplete, ToCopyableBuilder { - private final TransferRequest request; + private final TransferObjectRequest request; private final TransferProgressSnapshot progressSnapshot; - private final CompletedTransfer completedTransfer; + private final CompletedObjectTransfer completedTransfer; private TransferListenerContext(Builder builder) { this.request = builder.request; @@ -56,7 +56,7 @@ public Builder toBuilder() { } @Override - public TransferRequest request() { + public TransferObjectRequest request() { return request; } @@ -66,7 +66,7 @@ public TransferProgressSnapshot progressSnapshot() { } @Override - public CompletedTransfer completedTransfer() { + public CompletedObjectTransfer completedTransfer() { return completedTransfer; } @@ -80,9 +80,9 @@ public String toString() { } public static final class Builder implements CopyableBuilder { - private TransferRequest request; + private TransferObjectRequest request; private TransferProgressSnapshot progressSnapshot; - private CompletedTransfer completedTransfer; + private CompletedObjectTransfer completedTransfer; private Builder() { super(); @@ -94,7 +94,7 @@ private Builder(TransferListenerContext context) { this.completedTransfer = context.completedTransfer; } - public Builder request(TransferRequest request) { + public Builder request(TransferObjectRequest request) { this.request = request; return this; } @@ -104,7 +104,7 @@ public Builder progressSnapshot(TransferProgressSnapshot progressSnapshot) { return this; } - public Builder completedTransfer(CompletedTransfer completedTransfer) { + public Builder completedTransfer(CompletedObjectTransfer completedTransfer) { this.completedTransfer = completedTransfer; return this; } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerFailedContext.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerFailedContext.java index 46a55cb5402a..cff4f8989965 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerFailedContext.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferListenerFailedContext.java @@ -18,7 +18,7 @@ import java.util.concurrent.CompletionException; import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.transfer.s3.TransferRequest; +import software.amazon.awssdk.transfer.s3.TransferObjectRequest; import software.amazon.awssdk.transfer.s3.progress.TransferListener; import software.amazon.awssdk.transfer.s3.progress.TransferProgressSnapshot; import software.amazon.awssdk.utils.ToString; @@ -62,7 +62,7 @@ public Builder toBuilder() { } @Override - public TransferRequest request() { + public TransferObjectRequest request() { return transferContext.request(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferProgressUpdater.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferProgressUpdater.java index cfc7469a35ca..a1114a96ad5b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferProgressUpdater.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/progress/TransferProgressUpdater.java @@ -24,10 +24,9 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.transfer.s3.CompletedTransfer; -import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.CompletedObjectTransfer; +import software.amazon.awssdk.transfer.s3.TransferObjectRequest; import software.amazon.awssdk.transfer.s3.TransferRequestOverrideConfiguration; -import software.amazon.awssdk.transfer.s3.UploadRequest; import software.amazon.awssdk.transfer.s3.internal.progress.NotifyingAsyncRequestBody.AsyncRequestBodyListener; import software.amazon.awssdk.transfer.s3.internal.progress.NotifyingAsyncResponseTransformer.AsyncResponseTransformerListener; import software.amazon.awssdk.transfer.s3.progress.TransferListener; @@ -44,7 +43,7 @@ public class TransferProgressUpdater { private final TransferListenerContext context; private final TransferListenerInvoker listeners; - public TransferProgressUpdater(UploadRequest request, AsyncRequestBody requestBody) { + public TransferProgressUpdater(TransferObjectRequest request, AsyncRequestBody requestBody) { DefaultTransferProgressSnapshot.Builder snapshotBuilder = DefaultTransferProgressSnapshot.builder(); getContentLengthSafe(requestBody).ifPresent(snapshotBuilder::transferSizeInBytes); TransferProgressSnapshot snapshot = snapshotBuilder.build(); @@ -58,18 +57,6 @@ public TransferProgressUpdater(UploadRequest request, AsyncRequestBody requestBo .orElseGet(Collections::emptyList)); } - public TransferProgressUpdater(DownloadRequest request) { - TransferProgressSnapshot snapshot = DefaultTransferProgressSnapshot.builder().build(); - progress = new DefaultTransferProgress(snapshot); - context = TransferListenerContext.builder() - .request(request) - .progressSnapshot(snapshot) - .build(); - listeners = new TransferListenerInvoker(request.overrideConfiguration() - .map(TransferRequestOverrideConfiguration::listeners) - .orElseGet(Collections::emptyList)); - } - public TransferProgress progress() { return progress; } @@ -97,11 +84,11 @@ public void beforeOnNext(ByteBuffer byteBuffer) { }); } - public AsyncResponseTransformer wrapResponseTransformer( - AsyncResponseTransformer responseTransformer) { + public AsyncResponseTransformer wrapResponseTransformer( + AsyncResponseTransformer responseTransformer) { return new NotifyingAsyncResponseTransformer<>( responseTransformer, - new AsyncResponseTransformerListener() { + new AsyncResponseTransformerListener() { @Override public void beforeOnResponse(GetObjectResponse response) { if (response.contentLength() != null) { @@ -124,7 +111,7 @@ public void beforeOnNext(ByteBuffer byteBuffer) { }); } - public void registerCompletion(CompletableFuture future) { + public void registerCompletion(CompletableFuture future) { future.whenComplete((r, t) -> { if (t == null) { listeners.transferComplete(context.copy(b -> { @@ -143,6 +130,9 @@ public void registerCompletion(CompletableFuture fu } private static Optional getContentLengthSafe(AsyncRequestBody requestBody) { + if (requestBody == null) { + return Optional.empty(); + } // requestBody.contentLength() may throw if the file does not exist. // We ignore any potential exception here to defer failure // to the s3CrtAsyncClient call and its associated future. diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferListener.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferListener.java index 8fcfd15c5103..925f77b1b8b8 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferListener.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferListener.java @@ -22,21 +22,22 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.transfer.s3.CompletedDownload; -import software.amazon.awssdk.transfer.s3.CompletedTransfer; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.CompletedObjectTransfer; +import software.amazon.awssdk.transfer.s3.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.TransferObjectRequest; import software.amazon.awssdk.transfer.s3.TransferRequest; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; /** * The {@link TransferListener} interface may be implemented by your application in order to receive event-driven updates on the - * progress of a transfer initiated by {@link S3TransferManager}. When you construct an {@link UploadRequest} or {@link - * DownloadRequest} request to submit to {@link S3TransferManager}, you may provide a variable number of {@link TransferListener}s - * to be associated with that request. Then, throughout the lifecycle of the request, {@link S3TransferManager} will invoke the - * provided {@link TransferListener}s when important events occur, like additional bytes being transferred, allowing you to - * monitor the ongoing progress of the transfer. + * progress of a transfer initiated by {@link S3TransferManager}. When you construct an {@link UploadFileRequest} or {@link + * DownloadFileRequest} request to submit to {@link S3TransferManager}, you may provide a variable number of {@link + * TransferListener}s to be associated with that request. Then, throughout the lifecycle of the request, {@link S3TransferManager} + * will invoke the provided {@link TransferListener}s when important events occur, like additional bytes being transferred, + * allowing you to monitor the ongoing progress of the transfer. *

* Each {@link TransferListener} callback is invoked with an immutable {@link Context} object. Depending on the current lifecycle * of the request, different {@link Context} objects have different attributes available (indicated by the provided context @@ -198,10 +199,10 @@ private Context() { @SdkPreviewApi public interface TransferInitiated { /** - * The {@link TransferRequest} that was submitted to {@link S3TransferManager}, i.e., the {@link UploadRequest} or - * {@link DownloadRequest}. + * The {@link TransferRequest} that was submitted to {@link S3TransferManager}, i.e., the {@link UploadFileRequest} or + * {@link DownloadFileRequest}. */ - TransferRequest request(); + TransferObjectRequest request(); /** * The immutable {@link TransferProgressSnapshot} for this specific update. @@ -241,9 +242,9 @@ public interface BytesTransferred extends TransferInitiated { @SdkPreviewApi public interface TransferComplete extends BytesTransferred { /** - * The completed transfer, i.e., the {@link CompletedUpload} or {@link CompletedDownload}. + * The completed transfer, i.e., the {@link CompletedFileUpload} or {@link CompletedFileDownload}. */ - CompletedTransfer completedTransfer(); + CompletedObjectTransfer completedTransfer(); } /** diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgress.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgress.java index 71e39ea3f739..e47415d0b0f7 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgress.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgress.java @@ -20,16 +20,16 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.Transfer; -import software.amazon.awssdk.transfer.s3.Upload; /** * {@link TransferProgress} is a stateful representation of the progress of a transfer initiated by {@link * S3TransferManager}. {@link TransferProgress} offers the ability to take a {@link #snapshot()} of the current progress, * represented by an immutable {@link TransferProgressSnapshot}, which contains helpful progress-related methods like {@link * TransferProgressSnapshot#bytesTransferred()} and {@link TransferProgressSnapshot#ratioTransferred()}. {@link TransferProgress} - * is attached to {@link Transfer} objects, namely {@link Upload} and {@link Download}. + * is attached to {@link Transfer} objects, namely {@link FileUpload} and {@link Download}. *

* Where possible, it is typically recommended to avoid directly querying {@link TransferProgress} and to instead leverage * the {@link TransferListener} interface to receive event-driven updates of the latest {@link TransferProgressSnapshot}. See the diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgressSnapshot.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgressSnapshot.java index 4299fb641e06..03eeb66750a2 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgressSnapshot.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/progress/TransferProgressSnapshot.java @@ -22,9 +22,9 @@ import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.TransferRequest; -import software.amazon.awssdk.transfer.s3.Upload; /** * {@link TransferProgressSnapshot} is an immutable, point-in-time representation of the progress of a given transfer @@ -32,7 +32,7 @@ * progress of a transfer, like {@link #bytesTransferred()} and {@link #ratioTransferred()}. *

* {@link TransferProgressSnapshot}'s methods that return {@link Optional} are dependent upon the size of a transfer (i.e., the - * {@code Content-Length}) being known. In the case of file-based {@link Upload}s, transfer sizes are known up front and + * {@code Content-Length}) being known. In the case of file-based {@link FileUpload}s, transfer sizes are known up front and * immediately available. In the case of {@link Download}s, the transfer size is not known until {@link S3TransferManager} * receives a {@link GetObjectResponse} from Amazon S3. *

@@ -59,7 +59,7 @@ public interface TransferProgressSnapshot { /** * The total size of the transfer, in bytes, or {@link Optional#empty()} if unknown. *

- * In the case of file-based {@link Upload}s, transfer sizes are known up front and immediately available. In the case of + * In the case of file-based {@link FileUpload}s, transfer sizes are known up front and immediately available. In the case of * {@link Download}s, the transfer size is not known until {@link S3TransferManager} receives a {@link GetObjectResponse} from * Amazon S3. */ diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java index a815e1ea4081..bfc90e8128cd 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java @@ -24,14 +24,14 @@ public class CompletedDownloadTest { @Test public void responseNull_shouldThrowException() { - assertThatThrownBy(() -> CompletedDownload.builder().build()).isInstanceOf(NullPointerException.class) - .hasMessageContaining("must not be null"); + assertThatThrownBy(() -> CompletedDownload.builder().result(null).build()).isInstanceOf(NullPointerException.class) + .hasMessageContaining("must not be null"); } @Test public void equalsHashcode() { EqualsVerifier.forClass(CompletedDownload.class) - .withNonnullFields("response") + .withNonnullFields("result") .verify(); } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileDownloadTest.java new file mode 100644 index 000000000000..1ff2e059f891 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileDownloadTest.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.Test; + +public class CompletedFileDownloadTest { + + @Test + public void responseNull_shouldThrowException() { + assertThatThrownBy(() -> CompletedFileDownload.builder().build()).isInstanceOf(NullPointerException.class) + .hasMessageContaining("must not be null"); + } + + @Test + public void equalsHashcode() { + EqualsVerifier.forClass(CompletedFileDownload.class) + .withNonnullFields("response") + .verify(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileUploadTest.java new file mode 100644 index 000000000000..807668c5012e --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedFileUploadTest.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.Test; + +public class CompletedFileUploadTest { + + @Test + public void responseNull_shouldThrowException() { + assertThatThrownBy(() -> CompletedFileUpload.builder().build()).isInstanceOf(NullPointerException.class) + .hasMessageContaining("must not be null"); + } + + @Test + public void equalsHashcode() { + EqualsVerifier.forClass(CompletedFileUpload.class) + .withNonnullFields("response") + .verify(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java index 2e79f3fbe39f..7f3738de6c4a 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java @@ -25,7 +25,7 @@ public class CompletedUploadTest { @Test public void responseNull_shouldThrowException() { assertThatThrownBy(() -> CompletedUpload.builder().build()).isInstanceOf(NullPointerException.class) - .hasMessageContaining("must not be null"); + .hasMessageContaining("must not be null"); } @Test diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadFileRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadFileRequestTest.java new file mode 100644 index 000000000000..7d59dfc43e78 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadFileRequestTest.java @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class DownloadFileRequestTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void noGetObjectRequest_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("getObjectRequest"); + + DownloadFileRequest.builder() + .destination(Paths.get(".")) + .build(); + } + + @Test + public void pathMissing_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("destination"); + + DownloadFileRequest.builder() + .getObjectRequest(b -> b.bucket("bucket").key("key")) + .build(); + } + + @Test + public void usingFile() { + Path path = Paths.get("."); + DownloadFileRequest requestUsingFile = DownloadFileRequest.builder() + .getObjectRequest(b -> b.bucket("bucket").key("key")) + .destination(path.toFile()) + .build(); + + assertThat(requestUsingFile.destination()).isEqualTo(path); + } + + @Test + public void usingFile_null_shouldThrowException() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("destination"); + File file = null; + DownloadFileRequest.builder() + .getObjectRequest(b -> b.bucket("bucket").key("key")) + .destination(file) + .build(); + + } + + @Test + public void equals_hashcode() { + EqualsVerifier.forClass(DownloadFileRequest.class) + .withNonnullFields("destination", "getObjectRequest") + .verify(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java index 6a54cd7e90e2..a83e73347a77 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java @@ -17,13 +17,13 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import nl.jqno.equalsverifier.EqualsVerifier; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; public class DownloadRequestTest { @@ -36,39 +36,32 @@ public void noGetObjectRequest_throws() { thrown.expectMessage("getObjectRequest"); DownloadRequest.builder() - .destination(Paths.get(".")) - .build(); - } - - @Test - public void pathMissing_throws() { - thrown.expect(NullPointerException.class); - thrown.expectMessage("destination"); - - DownloadRequest.builder() - .getObjectRequest(b -> b.bucket("bucket").key("key")) + .responseTransformer(AsyncResponseTransformer.toBytes()) .build(); } @Test public void usingFile() { - Path path = Paths.get("."); + AsyncResponseTransformer> responseTransformer = + AsyncResponseTransformer.toBytes(); + DownloadRequest requestUsingFile = DownloadRequest.builder() .getObjectRequest(b -> b.bucket("bucket").key("key")) - .destination(path.toFile()) + .responseTransformer(responseTransformer) .build(); - assertThat(requestUsingFile.destination()).isEqualTo(path); + assertThat(requestUsingFile.responseTransformer()).isEqualTo(responseTransformer); } + @Test - public void usingFile_null_shouldThrowException() { + public void null_responseTransformer_shouldThrowException() { thrown.expect(NullPointerException.class); - thrown.expectMessage("destination"); - File file = null; + thrown.expectMessage("responseTransformer"); + DownloadRequest.builder() .getObjectRequest(b -> b.bucket("bucket").key("key")) - .destination(file) + .responseTransformer(null) .build(); } @@ -76,7 +69,7 @@ public void usingFile_null_shouldThrowException() { @Test public void equals_hashcode() { EqualsVerifier.forClass(DownloadRequest.class) - .withNonnullFields("destination", "getObjectRequest") + .withNonnullFields("responseTransformer", "getObjectRequest") .verify(); } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedFileUploadTest.java similarity index 88% rename from services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java rename to services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedFileUploadTest.java index a2bd8112b9e4..828e1e8ced40 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedFileUploadTest.java @@ -22,7 +22,7 @@ import org.junit.Test; import software.amazon.awssdk.core.exception.SdkClientException; -public class FailedSingleFileUploadTest { +public class FailedFileUploadTest { @Test public void requestNull_mustThrowException() { @@ -34,10 +34,10 @@ public void requestNull_mustThrowException() { @Test public void exceptionNull_mustThrowException() { - UploadRequest uploadRequest = - UploadRequest.builder().source(Paths.get(".")).putObjectRequest(p -> p.bucket("bucket").key("key")).build(); + UploadFileRequest uploadFileRequest = + UploadFileRequest.builder().source(Paths.get(".")).putObjectRequest(p -> p.bucket("bucket").key("key")).build(); assertThatThrownBy(() -> FailedFileUpload.builder() - .request(uploadRequest).build()) + .request(uploadFileRequest).build()) .isInstanceOf(NullPointerException.class) .hasMessageContaining("exception must not be null"); } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadFileRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadFileRequestTest.java new file mode 100644 index 000000000000..6fea04afe0b0 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadFileRequestTest.java @@ -0,0 +1,80 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +public class UploadFileRequestTest { + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void upload_noRequestParamsProvided_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("putObjectRequest"); + + UploadFileRequest.builder() + .source(Paths.get(".")) + .build(); + } + + @Test + public void pathMissing_shouldThrow() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("source"); + UploadFileRequest.builder() + .putObjectRequest(PutObjectRequest.builder().build()) + .build(); + } + + @Test + public void sourceUsingFile() { + Path path = Paths.get("."); + UploadFileRequest request = UploadFileRequest.builder() + .putObjectRequest(b -> b.bucket("bucket").key("key")) + .source(path.toFile()) + .build(); + assertThat(request.source()).isEqualTo(path); + } + + @Test + public void sourceUsingFile_null_shouldThrowException() { + File file = null; + thrown.expect(NullPointerException.class); + thrown.expectMessage("source"); + UploadFileRequest.builder() + .putObjectRequest(b -> b.bucket("bucket").key("key")) + .source(file) + .build(); + } + + @Test + public void equals_hashcode() { + EqualsVerifier.forClass(UploadFileRequest.class) + .withNonnullFields("source", "putObjectRequest") + .verify(); + } + +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java index 08debe7b8c1a..5fc97c2c8312 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java @@ -17,13 +17,11 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import nl.jqno.equalsverifier.EqualsVerifier; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.model.PutObjectRequest; public class UploadRequestTest { @@ -36,44 +34,43 @@ public void upload_noRequestParamsProvided_throws() { thrown.expectMessage("putObjectRequest"); UploadRequest.builder() - .source(Paths.get(".")) + .requestBody(AsyncRequestBody.fromString("foo")) .build(); } @Test - public void pathMissing_shouldThrow() { + public void bodyMissing_shouldThrow() { thrown.expect(NullPointerException.class); - thrown.expectMessage("source"); + thrown.expectMessage("requestBody"); UploadRequest.builder() .putObjectRequest(PutObjectRequest.builder().build()) .build(); } @Test - public void sourceUsingFile() { - Path path = Paths.get("."); + public void bodyEqualsGivenBody() { + AsyncRequestBody requestBody = AsyncRequestBody.fromString("foo"); UploadRequest request = UploadRequest.builder() .putObjectRequest(b -> b.bucket("bucket").key("key")) - .source(path.toFile()) + .requestBody(requestBody) .build(); - assertThat(request.source()).isEqualTo(path); + assertThat(request.requestBody()).isSameAs(requestBody); } @Test - public void sourceUsingFile_null_shouldThrowException() { - File file = null; + public void null_requestBody_shouldThrowException() { thrown.expect(NullPointerException.class); - thrown.expectMessage("source"); + thrown.expectMessage("requestBody"); UploadRequest.builder() .putObjectRequest(b -> b.bucket("bucket").key("key")) - .source(file) + .requestBody(null) .build(); } @Test public void equals_hashcode() { EqualsVerifier.forClass(UploadRequest.class) - .withNonnullFields("source", "putObjectRequest") + .withNonnullFields("requestBody", "putObjectRequest") .verify(); } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java similarity index 76% rename from services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java rename to services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java index 64da9ab51145..bfe9dbd438c7 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java @@ -17,14 +17,14 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.junit.Test; -import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory; +import software.amazon.awssdk.transfer.s3.CompletedDirectoryUpload; -public class CompletedUploadDirectoryTest { +public class CompletedDirectoryUploadTest { @Test public void equalsHashcode() { - EqualsVerifier.forClass(CompletedUploadDirectory.class) - .withNonnullFields("failedUploads") + EqualsVerifier.forClass(CompletedDirectoryUpload.class) + .withNonnullFields("failedTransfers") .verify(); } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java index eaa8a73d5379..8a08e422c438 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java @@ -49,12 +49,12 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.Download; -import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.DownloadFileRequest; +import software.amazon.awssdk.transfer.s3.FileDownload; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.S3TransferManager; -import software.amazon.awssdk.transfer.s3.Upload; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; import software.amazon.awssdk.transfer.s3.progress.TransferListener; public class S3TransferManagerListenerTest { @@ -86,19 +86,19 @@ public void upload_success_shouldInvokeListener() throws Exception { Path path = newTempFile(); Files.write(path, randomBytes(contentLength)); - UploadRequest uploadRequest = UploadRequest.builder() - .putObjectRequest(r -> r.bucket("bucket") - .key("key")) - .source(path) - .overrideConfiguration(b -> b.addListener(listener)) - .build(); - Upload upload = tm.upload(uploadRequest); + UploadFileRequest uploadFileRequest = UploadFileRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") + .key("key")) + .source(path) + .overrideConfiguration(b -> b.addListener(listener)) + .build(); + FileUpload fileUpload = tm.uploadFile(uploadFileRequest); ArgumentCaptor captor1 = ArgumentCaptor.forClass(TransferListener.Context.TransferInitiated.class); verify(listener, timeout(1000).times(1)).transferInitiated(captor1.capture()); TransferListener.Context.TransferInitiated ctx1 = captor1.getValue(); - assertThat(ctx1.request()).isSameAs(uploadRequest); + assertThat(ctx1.request()).isSameAs(uploadFileRequest); assertThat(ctx1.progressSnapshot().transferSizeInBytes()).hasValue(contentLength); assertThat(ctx1.progressSnapshot().bytesTransferred()).isZero(); @@ -106,7 +106,7 @@ public void upload_success_shouldInvokeListener() throws Exception { ArgumentCaptor.forClass(TransferListener.Context.BytesTransferred.class); verify(listener, timeout(1000).times(1)).bytesTransferred(captor2.capture()); TransferListener.Context.BytesTransferred ctx2 = captor2.getValue(); - assertThat(ctx2.request()).isSameAs(uploadRequest); + assertThat(ctx2.request()).isSameAs(uploadFileRequest); assertThat(ctx2.progressSnapshot().transferSizeInBytes()).hasValue(contentLength); assertThat(ctx2.progressSnapshot().bytesTransferred()).isPositive(); @@ -114,12 +114,12 @@ public void upload_success_shouldInvokeListener() throws Exception { ArgumentCaptor.forClass(TransferListener.Context.TransferComplete.class); verify(listener, timeout(1000).times(1)).transferComplete(captor3.capture()); TransferListener.Context.TransferComplete ctx3 = captor3.getValue(); - assertThat(ctx3.request()).isSameAs(uploadRequest); + assertThat(ctx3.request()).isSameAs(uploadFileRequest); assertThat(ctx3.progressSnapshot().transferSizeInBytes()).hasValue(contentLength); assertThat(ctx3.progressSnapshot().bytesTransferred()).isEqualTo(contentLength); - assertThat(ctx3.completedTransfer()).isSameAs(upload.completionFuture().get()); + assertThat(ctx3.completedTransfer()).isSameAs(fileUpload.completionFuture().get()); - upload.completionFuture().join(); + fileUpload.completionFuture().join(); verifyNoMoreInteractions(listener); } @@ -127,13 +127,13 @@ public void upload_success_shouldInvokeListener() throws Exception { public void download_success_shouldInvokeListener() throws Exception { TransferListener listener = mock(TransferListener.class); - DownloadRequest downloadRequest = DownloadRequest.builder() - .getObjectRequest(r -> r.bucket("bucket") - .key("key")) - .destination(newTempFile()) - .overrideConfiguration(b -> b.addListener(listener)) - .build(); - Download download = tm.download(downloadRequest); + DownloadFileRequest downloadRequest = DownloadFileRequest.builder() + .getObjectRequest(r -> r.bucket("bucket") + .key("key")) + .destination(newTempFile()) + .overrideConfiguration(b -> b.addListener(listener)) + .build(); + FileDownload download = tm.downloadFile(downloadRequest); ArgumentCaptor captor1 = ArgumentCaptor.forClass(TransferListener.Context.TransferInitiated.class); @@ -173,15 +173,15 @@ public void upload_failure_shouldInvokeListener() throws Exception { Path path = newTempFile(); Files.write(path, randomBytes(contentLength)); - UploadRequest uploadRequest = UploadRequest.builder() - .putObjectRequest(r -> r.bucket("bucket") - .key("key")) - .source(Paths.get("/some/nonexistent/path")) - .overrideConfiguration(b -> b.addListener(listener)) - .build(); - Upload upload = tm.upload(uploadRequest); + UploadFileRequest uploadFileRequest = UploadFileRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") + .key("key")) + .source(Paths.get("/some/nonexistent/path")) + .overrideConfiguration(b -> b.addListener(listener)) + .build(); + FileUpload fileUpload = tm.uploadFile(uploadFileRequest); - CompletableFuture future = upload.completionFuture(); + CompletableFuture future = fileUpload.completionFuture(); assertThatThrownBy(future::join) .isInstanceOf(CompletionException.class) .hasCauseInstanceOf(NoSuchFileException.class); @@ -190,7 +190,7 @@ public void upload_failure_shouldInvokeListener() throws Exception { ArgumentCaptor.forClass(TransferListener.Context.TransferInitiated.class); verify(listener, timeout(1000).times(1)).transferInitiated(captor1.capture()); TransferListener.Context.TransferInitiated ctx1 = captor1.getValue(); - assertThat(ctx1.request()).isSameAs(uploadRequest); + assertThat(ctx1.request()).isSameAs(uploadFileRequest); // transferSize is not known since file did not exist assertThat(ctx1.progressSnapshot().transferSizeInBytes()).isNotPresent(); assertThat(ctx1.progressSnapshot().bytesTransferred()).isZero(); @@ -199,12 +199,11 @@ public void upload_failure_shouldInvokeListener() throws Exception { ArgumentCaptor.forClass(TransferListener.Context.TransferFailed.class); verify(listener, timeout(1000).times(1)).transferFailed(captor2.capture()); TransferListener.Context.TransferFailed ctx2 = captor2.getValue(); - assertThat(ctx2.request()).isSameAs(uploadRequest); + assertThat(ctx2.request()).isSameAs(uploadFileRequest); assertThat(ctx2.progressSnapshot().transferSizeInBytes()).isNotPresent(); assertThat(ctx2.progressSnapshot().bytesTransferred()).isZero(); assertThat(ctx2.exception()).isInstanceOf(NoSuchFileException.class); - upload.completionFuture().join(); verifyNoMoreInteractions(listener); } @@ -215,19 +214,19 @@ public void listener_exception_shouldBeSuppressed() throws Exception { Path path = newTempFile(); Files.write(path, randomBytes(contentLength)); - UploadRequest uploadRequest = UploadRequest.builder() - .putObjectRequest(r -> r.bucket("bucket") - .key("key")) - .source(path) - .overrideConfiguration(b -> b.addListener(listener)) - .build(); - Upload upload = tm.upload(uploadRequest); + UploadFileRequest uploadFileRequest = UploadFileRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") + .key("key")) + .source(path) + .overrideConfiguration(b -> b.addListener(listener)) + .build(); + FileUpload fileUpload = tm.uploadFile(uploadFileRequest); verify(listener, timeout(1000).times(1)).transferInitiated(any()); verify(listener, timeout(1000).times(1)).bytesTransferred(any()); verify(listener, timeout(1000).times(1)).transferComplete(any()); - upload.completionFuture().join(); + fileUpload.completionFuture().join(); verifyNoMoreInteractions(listener); } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java index e018b645816e..79659cb1459f 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java @@ -33,12 +33,12 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.transfer.s3.CompletedDownload; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; public class S3TransferManagerTest { private S3CrtAsyncClient mockS3Crt; @@ -71,15 +71,15 @@ public void upload_returnsResponse() { when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) .thenReturn(CompletableFuture.completedFuture(response)); - CompletedUpload completedUpload = tm.upload(UploadRequest.builder() - .putObjectRequest(r -> r.bucket("bucket") + CompletedFileUpload completedFileUpload = tm.uploadFile(UploadFileRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") .key("key")) - .source(Paths.get(".")) - .build()) - .completionFuture() - .join(); + .source(Paths.get(".")) + .build()) + .completionFuture() + .join(); - assertThat(completedUpload.response()).isEqualTo(response); + assertThat(completedFileUpload.response()).isEqualTo(response); } @Test @@ -88,12 +88,12 @@ public void upload_cancel_shouldForwardCancellation() { when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) .thenReturn(s3CrtFuture); - CompletableFuture future = tm.upload(UploadRequest.builder() - .putObjectRequest(r -> r.bucket("bucket") + CompletableFuture future = tm.uploadFile(UploadFileRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") .key("key")) - .source(Paths.get(".")) - .build()) - .completionFuture(); + .source(Paths.get(".")) + .build()) + .completionFuture(); future.cancel(true); assertThat(s3CrtFuture).isCancelled(); @@ -105,14 +105,14 @@ public void download_returnsResponse() { when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .thenReturn(CompletableFuture.completedFuture(response)); - CompletedDownload completedDownload = tm.download(DownloadRequest.builder() - .getObjectRequest(r -> r.bucket("bucket") + CompletedFileDownload completedFileDownload = tm.downloadFile(DownloadFileRequest.builder() + .getObjectRequest(r -> r.bucket("bucket") .key("key")) - .destination(Paths.get(".")) - .build()) - .completionFuture() - .join(); - assertThat(completedDownload.response()).isEqualTo(response); + .destination(Paths.get(".")) + .build()) + .completionFuture() + .join(); + assertThat(completedFileDownload.response()).isEqualTo(response); } @Test @@ -121,12 +121,12 @@ public void download_cancel_shouldForwardCancellation() { when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .thenReturn(s3CrtFuture); - CompletableFuture future = tm.download(DownloadRequest.builder() - .getObjectRequest(r -> r.bucket("bucket") + CompletableFuture future = tm.downloadFile(DownloadFileRequest.builder() + .getObjectRequest(r -> r.bucket("bucket") .key("key")) - .destination(Paths.get(".")) - .build()) - .completionFuture(); + .destination(Paths.get(".")) + .build()) + .completionFuture(); future.cancel(true); assertThat(s3CrtFuture).isCancelled(); } @@ -134,13 +134,13 @@ public void download_cancel_shouldForwardCancellation() { @Test public void objectLambdaArnBucketProvided_shouldThrowException() { String objectLambdaArn = "arn:xxx:s3-object-lambda"; - assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn) - .key("key")).source(Paths.get("."))) + assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn) + .key("key")).source(Paths.get("."))) .completionFuture().join()) .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn) - .key("key")).destination(Paths.get("."))).completionFuture().join()) + assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn) + .key("key")).destination(Paths.get("."))).completionFuture().join()) .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(objectLambdaArn).sourceDirectory(Paths.get("."))).completionFuture().join()) @@ -150,13 +150,13 @@ public void objectLambdaArnBucketProvided_shouldThrowException() { @Test public void mrapArnProvided_shouldThrowException() { String mrapArn = "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"; - assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(mrapArn) - .key("key")).source(Paths.get("."))) + assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(mrapArn) + .key("key")).source(Paths.get("."))) .completionFuture().join()) .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(mrapArn) - .key("key")).destination(Paths.get("."))).completionFuture().join()) + assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(mrapArn) + .key("key")).destination(Paths.get("."))).completionFuture().join()) .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(mrapArn).sourceDirectory(Paths.get("."))).completionFuture().join()) @@ -191,15 +191,15 @@ public void uploadDirectory_requestNull_shouldThrowException() { @Test public void upload_requestNull_shouldThrowException() { - UploadRequest request = null; - assertThatThrownBy(() -> tm.upload(request).completionFuture().join()).isInstanceOf(NullPointerException.class) - .hasMessageContaining("must not be null"); + UploadFileRequest request = null; + assertThatThrownBy(() -> tm.uploadFile(request).completionFuture().join()).isInstanceOf(NullPointerException.class) + .hasMessageContaining("must not be null"); } @Test public void download_requestNull_shouldThrowException() { - DownloadRequest request = null; - assertThatThrownBy(() -> tm.download(request).completionFuture().join()).isInstanceOf(NullPointerException.class) - .hasMessageContaining("must not be null"); + DownloadFileRequest request = null; + assertThatThrownBy(() -> tm.downloadFile(request).completionFuture().join()).isInstanceOf(NullPointerException.class) + .hasMessageContaining("must not be null"); } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java index 25538052a3a8..8e361f579ba8 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java @@ -45,11 +45,11 @@ import org.mockito.ArgumentCaptor; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.testutils.FileUtils; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.Upload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.DirectoryUpload; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest; -import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgress; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgressSnapshot; import software.amazon.awssdk.utils.IoUtils; @@ -63,7 +63,7 @@ public class UploadDirectoryHelperParameterizedTest { Configuration.osX(), Configuration.windows(), Configuration.forCurrentPlatform())); - private Function singleUploadFunction; + private Function singleUploadFunction; private UploadDirectoryHelper uploadDirectoryHelper; private Path directory; @@ -101,11 +101,11 @@ public void tearDown() { @Test public void uploadDirectory_defaultSetting_shouldRecursivelyUpload() { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())) .thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -113,7 +113,7 @@ public void uploadDirectory_defaultSetting_shouldRecursivelyUpload() { .build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket")); assertThat(actualRequests.size()).isEqualTo(3); @@ -127,10 +127,10 @@ public void uploadDirectory_defaultSetting_shouldRecursivelyUpload() { @Test public void uploadDirectory_recursiveFalse_shouldOnlyUploadTopLevel() { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -138,7 +138,7 @@ public void uploadDirectory_recursiveFalse_shouldOnlyUploadTopLevel() { .build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); assertThat(actualRequests.size()).isEqualTo(1); @@ -150,10 +150,10 @@ public void uploadDirectory_recursiveFalse_shouldOnlyUploadTopLevel() { public void uploadDirectory_recursiveFalseFollowSymlinkTrue_shouldOnlyUploadTopLevel() { // skip the test if we are using jimfs because it doesn't work well with symlink assumeTrue(configuration.equals(Configuration.forCurrentPlatform())); - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -161,7 +161,7 @@ public void uploadDirectory_recursiveFalseFollowSymlinkTrue_shouldOnlyUploadTopL .build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); List keys = actualRequests.stream().map(u -> u.putObjectRequest().key()) .collect(Collectors.toList()); @@ -175,10 +175,10 @@ public void uploadDirectory_FollowSymlinkTrue_shouldIncludeLinkedFiles() { // skip the test if we are using jimfs because it doesn't work well with symlink assumeTrue(configuration.equals(Configuration.forCurrentPlatform())); - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -186,7 +186,7 @@ public void uploadDirectory_FollowSymlinkTrue_shouldIncludeLinkedFiles() { .build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket")); List keys = @@ -199,10 +199,10 @@ public void uploadDirectory_FollowSymlinkTrue_shouldIncludeLinkedFiles() { @Test public void uploadDirectory_withPrefix_keysShouldHavePrefix() { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -220,10 +220,10 @@ public void uploadDirectory_withPrefix_keysShouldHavePrefix() { @Test public void uploadDirectory_withDelimiter_shouldHonor() { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -242,11 +242,11 @@ public void uploadDirectory_withDelimiter_shouldHonor() { @Test public void uploadDirectory_maxLengthOne_shouldOnlyUploadTopLevel() { - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())) .thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -254,7 +254,7 @@ public void uploadDirectory_maxLengthOne_shouldOnlyUploadTopLevel() { .build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket")); assertThat(actualRequests.size()).isEqualTo(1); @@ -288,17 +288,17 @@ public void uploadDirectory_notDirectory_shouldCompleteFutureExceptionally() { public void uploadDirectory_notDirectoryFollowSymlinkTrue_shouldCompleteSuccessfully() { // skip the test if we are using jimfs because it doesn't work well with symlink assumeTrue(configuration.equals(Configuration.forCurrentPlatform())); - ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class); + ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadFileRequest.class); when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload()); - UploadDirectoryTransfer uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() - .overrideConfiguration(o -> o.followSymbolicLinks(true)) - .sourceDirectory(Paths.get(directory.toString(), "symlink")) - .bucket("bucket").build()); + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() + .overrideConfiguration(o -> o.followSymbolicLinks(true)) + .sourceDirectory(Paths.get(directory.toString(), "symlink")) + .bucket("bucket").build()); uploadDirectory.completionFuture().join(); - List actualRequests = requestArgumentCaptor.getAllValues(); + List actualRequests = requestArgumentCaptor.getAllValues(); actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket")); assertThat(actualRequests.size()).isEqualTo(1); @@ -310,10 +310,10 @@ public void uploadDirectory_notDirectoryFollowSymlinkTrue_shouldCompleteSuccessf assertThat(keys).containsOnly("2.txt"); } - private DefaultUpload completedUpload() { - return new DefaultUpload(CompletableFuture.completedFuture(CompletedUpload.builder() - .response(PutObjectResponse.builder().build()) - .build()), + private DefaultFileUpload completedUpload() { + return new DefaultFileUpload(CompletableFuture.completedFuture(CompletedFileUpload.builder() + .response(PutObjectResponse.builder().build()) + .build()), new DefaultTransferProgress(DefaultTransferProgressSnapshot.builder().build())); } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java index e95fca2288f3..51d242fb26c5 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java @@ -38,19 +38,19 @@ import org.junit.Test; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.transfer.s3.CompletedUpload; -import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory; -import software.amazon.awssdk.transfer.s3.Upload; +import software.amazon.awssdk.transfer.s3.CompletedDirectoryUpload; +import software.amazon.awssdk.transfer.s3.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.DirectoryUpload; +import software.amazon.awssdk.transfer.s3.FileUpload; import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest; -import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer; -import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgress; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgressSnapshot; public class UploadDirectoryHelperTest { private static FileSystem jimfs; private static Path directory; - private Function singleUploadFunction; + private Function singleUploadFunction; private UploadDirectoryHelper uploadDirectoryHelper; @BeforeClass @@ -75,15 +75,15 @@ public void methodSetup() { @Test public void uploadDirectory_cancel_shouldCancelAllFutures() { - CompletableFuture future = new CompletableFuture<>(); - Upload upload = newUpload(future); + CompletableFuture future = new CompletableFuture<>(); + FileUpload fileUpload = newUpload(future); - CompletableFuture future2 = new CompletableFuture<>(); - Upload upload2 = newUpload(future2); + CompletableFuture future2 = new CompletableFuture<>(); + FileUpload fileUpload2 = newUpload(future2); - when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2); + when(singleUploadFunction.apply(any(UploadFileRequest.class))).thenReturn(fileUpload, fileUpload2); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") @@ -102,62 +102,62 @@ public void uploadDirectory_cancel_shouldCancelAllFutures() { public void uploadDirectory_allUploadsSucceed_failedUploadsShouldBeEmpty() throws ExecutionException, InterruptedException, TimeoutException { PutObjectResponse putObjectResponse = PutObjectResponse.builder().eTag("1234").build(); - CompletedUpload completedUpload = CompletedUpload.builder().response(putObjectResponse).build(); - CompletableFuture successfulFuture = new CompletableFuture<>(); + CompletedFileUpload completedFileUpload = CompletedFileUpload.builder().response(putObjectResponse).build(); + CompletableFuture successfulFuture = new CompletableFuture<>(); - Upload upload = newUpload(successfulFuture); - successfulFuture.complete(completedUpload); + FileUpload fileUpload = newUpload(successfulFuture); + successfulFuture.complete(completedFileUpload); PutObjectResponse putObjectResponse2 = PutObjectResponse.builder().eTag("5678").build(); - CompletedUpload completedUpload2 = CompletedUpload.builder().response(putObjectResponse2).build(); - CompletableFuture failedFuture = new CompletableFuture<>(); - Upload upload2 = newUpload(failedFuture); - failedFuture.complete(completedUpload2); + CompletedFileUpload completedFileUpload2 = CompletedFileUpload.builder().response(putObjectResponse2).build(); + CompletableFuture failedFuture = new CompletableFuture<>(); + FileUpload fileUpload2 = newUpload(failedFuture); + failedFuture.complete(completedFileUpload2); - when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2); + when(singleUploadFunction.apply(any(UploadFileRequest.class))).thenReturn(fileUpload, fileUpload2); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") .build()); - CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS); + CompletedDirectoryUpload completedDirectoryUpload = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS); - assertThat(completedUploadDirectory.failedUploads()).isEmpty(); + assertThat(completedDirectoryUpload.failedTransfers()).isEmpty(); } @Test public void uploadDirectory_partialSuccess_shouldProvideFailedUploads() throws ExecutionException, InterruptedException, TimeoutException { PutObjectResponse putObjectResponse = PutObjectResponse.builder().eTag("1234").build(); - CompletedUpload completedUpload = CompletedUpload.builder().response(putObjectResponse).build(); - CompletableFuture successfulFuture = new CompletableFuture<>(); - Upload upload = newUpload(successfulFuture); - successfulFuture.complete(completedUpload); + CompletedFileUpload completedFileUpload = CompletedFileUpload.builder().response(putObjectResponse).build(); + CompletableFuture successfulFuture = new CompletableFuture<>(); + FileUpload fileUpload = newUpload(successfulFuture); + successfulFuture.complete(completedFileUpload); SdkClientException exception = SdkClientException.create("failed"); - CompletableFuture failedFuture = new CompletableFuture<>(); - Upload upload2 = newUpload(failedFuture); + CompletableFuture failedFuture = new CompletableFuture<>(); + FileUpload fileUpload2 = newUpload(failedFuture); failedFuture.completeExceptionally(exception); - when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2); + when(singleUploadFunction.apply(any(UploadFileRequest.class))).thenReturn(fileUpload, fileUpload2); - UploadDirectoryTransfer uploadDirectory = + DirectoryUpload uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder() .sourceDirectory(directory) .bucket("bucket") .build()); - CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS); + CompletedDirectoryUpload completedDirectoryUpload = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS); - assertThat(completedUploadDirectory.failedUploads()).hasSize(1); - assertThat(completedUploadDirectory.failedUploads().iterator().next().exception()).isEqualTo(exception); - assertThat(completedUploadDirectory.failedUploads().iterator().next().request().source().toString()).isEqualTo("test/2"); + assertThat(completedDirectoryUpload.failedTransfers()).hasSize(1); + assertThat(completedDirectoryUpload.failedTransfers().iterator().next().exception()).isEqualTo(exception); + assertThat(completedDirectoryUpload.failedTransfers().iterator().next().request().source().toString()).isEqualTo("test/2"); } - private Upload newUpload(CompletableFuture future) { - return new DefaultUpload(future, + private FileUpload newUpload(CompletableFuture future) { + return new DefaultFileUpload(future, new DefaultTransferProgress(DefaultTransferProgressSnapshot.builder().build()) ); } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/progress/LoggingTransferListenerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/progress/LoggingTransferListenerTest.java index 930b6094bdca..2ea46b74fe85 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/progress/LoggingTransferListenerTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/progress/LoggingTransferListenerTest.java @@ -24,8 +24,8 @@ import org.junit.Before; import org.junit.Test; import software.amazon.awssdk.testutils.LogCaptor; -import software.amazon.awssdk.transfer.s3.CompletedTransfer; -import software.amazon.awssdk.transfer.s3.TransferRequest; +import software.amazon.awssdk.transfer.s3.CompletedObjectTransfer; +import software.amazon.awssdk.transfer.s3.TransferObjectRequest; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgress; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgressSnapshot; import software.amazon.awssdk.transfer.s3.internal.progress.TransferListenerContext; @@ -45,7 +45,7 @@ public void setUp() throws Exception { .build(); progress = new DefaultTransferProgress(snapshot); context = TransferListenerContext.builder() - .request(mock(TransferRequest.class)) + .request(mock(TransferObjectRequest.class)) .progressSnapshot(snapshot) .build(); listener = LoggingTransferListener.create(); @@ -111,7 +111,7 @@ private void invokeSuccessfulLifecycle() { } listener.transferComplete(context.copy(b -> b.progressSnapshot(progress.snapshot()) - .completedTransfer(mock(CompletedTransfer.class)))); + .completedTransfer(mock(CompletedObjectTransfer.class)))); } private void assertLogged(List events, Level level, String message) { diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java index df0f93ed9420..25f299e744e8 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java @@ -38,6 +38,14 @@ public static byte[] computeCheckSum(InputStream is) throws IOException { return instance.digest(); } + public static byte[] computeCheckSum(byte[] bytes) { + MessageDigest instance = createMessageDigest(); + + instance.update(bytes); + + return instance.digest(); + } + public static byte[] computeCheckSum(ByteBuffer bb) { MessageDigest instance = createMessageDigest(); diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java index 9126c48ee2ec..3d1c04d58a0d 100644 --- a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.transfer.s3.FileDownload; import software.amazon.awssdk.utils.Logger; public class TransferManagerDownloadBenchmark extends BaseTransferManagerBenchmark { @@ -75,9 +75,9 @@ private void downloadToFile(int count, boolean printoutResult) { private void downloadOnceToFile(List latencies) { Path downloadPath = new File(this.path).toPath(); long start = System.currentTimeMillis(); - Download download = - transferManager.download(b -> b.getObjectRequest(r -> r.bucket(bucket).key(key)) - .destination(downloadPath)); + FileDownload download = + transferManager.downloadFile(b -> b.getObjectRequest(r -> r.bucket(bucket).key(key)) + .destination(downloadPath)); download.completionFuture().join(); long end = System.currentTimeMillis(); latencies.add((end - start) / 1000.0); diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java index 39e9bf69ff4b..ac987aa33f61 100644 --- a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java @@ -55,8 +55,8 @@ private void uploadFromFile(int count, boolean printOutResult) { private void uploadOnceFromFile(List latencies) { File sourceFile = new File(path); long start = System.currentTimeMillis(); - transferManager.upload(b -> b.putObjectRequest(r -> r.bucket(bucket).key(key)) - .source(sourceFile.toPath())) + transferManager.uploadFile(b -> b.putObjectRequest(r -> r.bucket(bucket).key(key)) + .source(sourceFile.toPath())) .completionFuture().join(); long end = System.currentTimeMillis(); latencies.add((end - start) / 1000.0); From ffc3debeefe93359b6548804b0694124d36955b8 Mon Sep 17 00:00:00 2001 From: Bennett Lynch Date: Wed, 3 Nov 2021 13:48:22 -0700 Subject: [PATCH 2/4] Incorporate feedback --- .../feature-AmazonS3-2a00881.json | 2 +- services-custom/s3-transfer-manager/README.md | 77 ++++++++++++++----- ...3TransferManagerUploadIntegrationTest.java | 9 +-- .../s3/CompletedDirectoryTransfer.java | 26 +------ .../transfer/s3/CompletedDirectoryUpload.java | 24 +++++- .../transfer/s3/CompletedFileUpload.java | 2 +- .../awssdk/transfer/s3/CompletedUpload.java | 2 +- .../awssdk/transfer/s3/FailedFileUpload.java | 6 +- .../awssdk/transfer/s3/S3TransferManager.java | 71 ++++++++--------- .../s3/internal/DefaultDirectoryUpload.java | 6 +- 10 files changed, 132 insertions(+), 93 deletions(-) diff --git a/.changes/next-release/feature-AmazonS3-2a00881.json b/.changes/next-release/feature-AmazonS3-2a00881.json index 9a47dc81dabc..3521ff768bca 100644 --- a/.changes/next-release/feature-AmazonS3-2a00881.json +++ b/.changes/next-release/feature-AmazonS3-2a00881.json @@ -2,5 +2,5 @@ "category": "Amazon S3", "contributor": "", "type": "feature", - "description": "Refactor S3TransferManager to support non-file-based transfers" + "description": "[Breaking Changes] Refactor S3TransferManager (PREVIEW) to support non-file-based transfers. This release refactors the S3TransferManager interface hierarchy and client API to differentiate between file-based and non-file-based transfers, allowing arbitrary object transfers. As a result, some S3TransferManager method signatures have changed in a backwards-incompatible way. Most notably, `Upload upload(UploadRequest)` becomes `FileUpload uploadFile(UploadFileRequest)`, and likewise for download variants. Please see https://github.com/aws/aws-sdk-java-v2/pull/2817 for a full list of changes." } diff --git a/services-custom/s3-transfer-manager/README.md b/services-custom/s3-transfer-manager/README.md index 6ac745a47481..81053d595985 100644 --- a/services-custom/s3-transfer-manager/README.md +++ b/services-custom/s3-transfer-manager/README.md @@ -19,47 +19,88 @@ First, you need to include the dependency in your project. ``` Note that you need to replace `${awsjavasdk.version}` with the latest -SDK version +SDK version. ### Instantiate the transfer manager -You can instantiate the transfer manager easily using the default settings -```java +You can instantiate the transfer manager easily using the default settings: -S3TransferManager transferManager = S3TransferManager.create(); - +```java +S3TransferManager tm = S3TransferManager.create(); ``` If you wish to configure settings, we recommend using the builder instead: + ```java -S3TransferManager transferManager = +S3TransferManager tm = S3TransferManager.builder() .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider) .region(Region.US_WEST_2) .targetThroughputInGbps(20.0) - .minimumPartSizeInBytes(10 * MB)) + .minimumPartSizeInBytes(8 * MB)) .build(); ``` +### Upload a file to S3 + +To upload a file to S3, you just need to provide the source file path and the `PutObjectRequest` that should be used for the upload: + +```java +FileUpload upload = + tm.uploadFile(u -> u.source(Paths.get("myFile.txt")) + .putObjectRequest(p -> p.bucket("bucket").key("key"))); +upload.completionFuture().join(); +``` + ### Download an S3 object to a file -To download an object, you just need to provide the destination file path and the `GetObjectRequest` that should be used for the download. + +To download an object, you just need to provide the destination file path and the `GetObjectRequest` that should be used for the download: ```java -Download download = - transferManager.download(b -> b.destination(path) - .getObjectRequest(r -> r.bucket("bucket") - .key("key"))); +FileDownload download = + tm.downloadFile(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key")) + .destination(Paths.get("myFile.txt"))); download.completionFuture().join(); ``` -### Upload a file to S3 -To upload a file to S3, you just need to provide the source file path and the `PutObjectRequest` that should be used for the upload. +### Upload any content to S3 -```java -Upload upload = transferManager.upload(b -> b.source(path) - .putObjectRequest(r -> r.bucket("bucket") - .key("key"))); +You may upload any arbitrary content to S3 by providing an `AsyncRequestBody`: +```java +Upload upload = + tm.upload(u -> u.requestBody(AsyncRequestBody.fromString("Hello world")) + .putObjectRequest(p -> p.bucket("bucket").key("key"))); upload.completionFuture().join(); +``` +Refer to the static factory methods available in `AsyncRequestBody` for other content sources. + +### Download an S3 object to a custom destination + +You may download an object from S3 to a custom destination by providing an `AsyncResponseTransformer`: + +*(This example buffers the entire object in memory and is not suitable for large objects.)* + +```java +Download> download = + tm.download(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key")) + .responseTransformer(AsyncResponseTransformer.toBytes())); +download.completionFuture().join(); +``` + +Refer to the static factory methods available in `AsyncResponseTransformer` for other destinations. + +### Attach a TransferListener + +To monitor a transfer's progress, you can include a `TransferListener` with your transfer request: + +```java +FileUpload upload = + tm.uploadFile(u -> u.source(Paths.get("myFile.txt")) + .putObjectRequest(p -> p.bucket("bucket").key("key")) + .overrideConfiguration(o -> o.addListener(LoggingTransferListener.create()))); +upload.completionFuture().join(); ``` + +You can provide your own implementation of a `TransferListener` to implement progress-bar-type functionality. \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java index cdfa315fe5ed..9125d0c1a416 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java @@ -67,11 +67,10 @@ public static void teardown() throws IOException { @Test public void upload_file_SentCorrectly() throws IOException { FileUpload fileUpload = - tm.uploadFile(UploadFileRequest.builder() - .putObjectRequest(b -> b.bucket(TEST_BUCKET).key(TEST_KEY)) - .source(testFile.toPath()) - .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - .build()); + tm.uploadFile(u -> u.putObjectRequest(p -> p.bucket(TEST_BUCKET).key(TEST_KEY)) + .source(testFile.toPath()) + .overrideConfiguration(o -> o.addListener(LoggingTransferListener.create())) + .build()); CompletedFileUpload completedFileUpload = fileUpload.completionFuture().join(); assertThat(completedFileUpload.response().responseMetadata().requestId()).isNotNull(); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java index 2cee56e09214..f0ab9be05caf 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryTransfer.java @@ -21,7 +21,7 @@ /** * A completed directory-based transfer. - * + * * @see CompletedDirectoryUpload */ @SdkPublicApi @@ -29,28 +29,10 @@ public interface CompletedDirectoryTransfer extends CompletedTransfer { /** - * An immutable collection of failed transfers with error details, request metadata about each file that is failed to - * transfer. - * - *

- * Failed single object transfers can be retried by calling {@link S3TransferManager#uploadFile(UploadFileRequest)} or - * {@link S3TransferManager#downloadFile(DownloadFileRequest)}. - * - * 

-     * {@code
-     * // Retrying failed uploads if the exception is retryable
-     * List> futures =
-     *     completedDirectoryUpload.failedTransfers()
-     *                             .stream()
-     *                             .filter(failedSingleFileUpload -> isRetryable(failedSingleFileUpload.exception()))
-     *                             .map(failedSingleFileUpload ->
-     *                                  tm.upload(failedSingleFileUpload.request()).completionFuture())
-     *                             .collect(Collectors.toList());
-     * CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
-     * }
-     *
+ * A list of failed transfer details, including the {@link FailedObjectTransfer#exception()} responsible for the failure and + * the {@link FailedObjectTransfer#request()} that initiated the transfer. * - * @return a list of failed transfers + * @return an immutable list of failed transfers */ Collection failedTransfers(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java index a39ffbb77812..b33d8f3b1fe7 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUpload.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.transfer.s3; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Objects; @@ -92,6 +93,14 @@ public interface Builder { */ Builder failedTransfers(Collection failedTransfers); + /** + * Add a {@link FailedFileUpload} + * + * @param failedTransfer failed upload + * @return This builder for method chaining. + */ + Builder addFailedTransfer(FailedFileUpload failedTransfer); + /** * Builds a {@link CompletedDirectoryUpload} based on the properties supplied to this builder * @return An initialized {@link CompletedDirectoryUpload} @@ -100,19 +109,28 @@ public interface Builder { } private static final class DefaultBuilder implements Builder { - private Collection failedTransfers = Collections.emptyList(); + private Collection failedTransfers; private DefaultBuilder() { } @Override public Builder failedTransfers(Collection failedTransfers) { - this.failedTransfers = failedTransfers; + this.failedTransfers = new ArrayList<>(failedTransfers); + return this; + } + + @Override + public Builder addFailedTransfer(FailedFileUpload failedTransfer) { + if (failedTransfers == null) { + failedTransfers = new ArrayList<>(); + } + failedTransfers.add(failedTransfer); return this; } public Collection getFailedTransfers() { - return failedTransfers; + return Collections.unmodifiableCollection(failedTransfers); } public void setFailedTransfers(Collection failedTransfers) { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java index 88cbe2ae913b..69642ef99ba0 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileUpload.java @@ -58,7 +58,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return response != null ? response.hashCode() : 0; + return response.hashCode(); } @Override diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java index c6f89a5ac951..9a8f0739b473 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java @@ -58,7 +58,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return response != null ? response.hashCode() : 0; + return response.hashCode(); } @Override diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java index 0587bda6620b..13deee4990f4 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedFileUpload.java @@ -107,9 +107,9 @@ private static final class DefaultBuilder implements Builder { private UploadFileRequest request; private Throwable exception; - private DefaultBuilder(FailedFileUpload failedSingleFileUpload) { - this.request = failedSingleFileUpload.request; - this.exception = failedSingleFileUpload.exception; + private DefaultBuilder(FailedFileUpload failedFileUpload) { + this.request = failedFileUpload.request; + this.exception = failedFileUpload.exception; } private DefaultBuilder() { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 864b4b5febfd..aa89840f9538 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -39,7 +39,7 @@ * // Create using all default configuration values * S3TransferManager tm = S3TransferManager.create(); * - * // If you wish to configure settings, we recommend using the builder instead: + * // If you wish to configure settings, we recommend using the builder instead * S3TransferManager tm = * S3TransferManager.builder() * .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider) @@ -48,22 +48,35 @@ * .minimumPartSizeInBytes(8 * MB)) * .build(); * + * // Upload a file to S3 + * FileUpload upload = + * tm.uploadFile(u -> u.source(Paths.get("myFile.txt")) + * .putObjectRequest(p -> p.bucket("bucket").key("key"))); + * upload.completionFuture().join(); + * * // Download an S3 object to a file * FileDownload download = - * tm.downloadFile(DownloadFileRequest.builder() - * .getObjectRequest(b -> b.bucket("bucket").key("key")) - * .destination(Paths.get("myFile.txt")) - * .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - * .build()); + * tm.downloadFile(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key")) + * .destination(Paths.get("myFile.txt"))); * download.completionFuture().join(); - * - * // Upload a file to S3 + * + * // Upload any content to S3 + * Upload upload = + * tm.upload(u -> u.requestBody(AsyncRequestBody.fromString("Hello world")) + * .putObjectRequest(p -> p.bucket("bucket").key("key"))); + * upload.completionFuture().join(); + * + * // Download an S3 object to a custom destination + * Download> download = + * tm.download(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key")) + * .responseTransformer(AsyncResponseTransformer.toBytes())); + * download.completionFuture().join(); + * + * // Attach a TransferListener * FileUpload upload = - * tm.uploadFile(UploadFileRequest.builder() - * .putObjectRequest(b -> b.bucket("bucket"").key("key"")) - * .source(Paths.get("myFile.txt")) - * .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - * .build()); + * tm.uploadFile(u -> u.source(Paths.get("myFile.txt")) + * .putObjectRequest(p -> p.bucket("bucket").key("key")) + * .overrideConfiguration(o -> o.addListener(LoggingTransferListener.create()))); * upload.completionFuture().join(); * } * @@ -80,11 +93,8 @@ public interface S3TransferManager extends SdkAutoCloseable { * {@code * // Initiate the transfer * FileDownload download = - * tm.downloadFile(DownloadFileRequest.builder() - * .getObjectRequest(b -> b.bucket("bucket").key("key")) - * .destination(Paths.get("myFile.txt")) - * .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create())) - * .build()); + * tm.downloadFile(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key")) + * .destination(Paths.get("myFile.txt"))); * // Wait for the transfer to complete * download.completionFuture().join(); * } @@ -110,17 +120,13 @@ default FileDownload downloadFile(Consumer request) /** * Download an object identified by the bucket and key from S3 through the given {@link AsyncResponseTransformer}. *

- * Usage Example (this example buffers the entire object into an in-memory byte array and is not suitable for - * large objects): + * Usage Example (this example buffers the entire object in memory and is not suitable for large objects): * 

      * {@code
      * // Initiate the transfer
      * Download> download =
-     *     tm.download(DownloadRequest.builder()
-     *                                .getObjectRequest(b -> b.bucket("bucket"").key("key"))
-     *                                .responseTransformer(AsyncResponseTransformer.toBytes())
-     *                                .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
-     *                                .build());
+     *     tm.download(d -> d.getObjectRequest(g -> g.bucket("bucket").key("key"))
+     *                       .responseTransformer(AsyncResponseTransformer.toBytes()));
      * // Wait for the transfer to complete
      * download.completionFuture().join();
      * }
@@ -155,11 +161,8 @@ default  Download download(Function
      * {@code
      * FileUpload upload =
-     *     tm.uploadFile(UploadFileRequest.builder()
-     *                                    .putObjectRequest(b -> b.bucket("bucket"").key("key""))
-     *                                    .source(Paths.get("myFile.txt"))
-     *                                    .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
-     *                                    .build());
+     *     tm.uploadFile(u -> u.source(Paths.get("myFile.txt"))
+     *                         .putObjectRequest(p -> p.bucket("bucket").key("key")));
      * // Wait for the transfer to complete
      * upload.completionFuture().join();
      * }
@@ -188,11 +191,9 @@ default FileUpload uploadFile(Consumer request) {
      * Usage Example:
      * 
      * {@code
-     * Upload upload = tm.upload(UploadRequest.builder()
-     *                                        .putObjectRequest(b -> b.bucket("bucket"").key("key""))
-     *                                        .requestBody(AsyncRequestBody.fromString("Hello world"))
-     *                                        .overrideConfiguration(b -> b.addListener(LoggingTransferListener.create()))
-     *                                        .build());
+     * Upload upload =
+     *     tm.upload(u -> u.requestBody(AsyncRequestBody.fromString("Hello world"))
+     *                     .putObjectRequest(p -> p.bucket("bucket").key("key")));
      * // Wait for the transfer to complete
      * upload.completionFuture().join();
      * }
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUpload.java
index c9f32063e5f1..a098d9b3ab8a 100644
--- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUpload.java
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUpload.java
@@ -17,14 +17,12 @@
 
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
-import software.amazon.awssdk.annotations.SdkPreviewApi;
-import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.transfer.s3.CompletedDirectoryUpload;
 import software.amazon.awssdk.transfer.s3.DirectoryUpload;
 import software.amazon.awssdk.utils.ToString;
 
-@SdkPublicApi
-@SdkPreviewApi
+@SdkInternalApi
 public final class DefaultDirectoryUpload implements DirectoryUpload {
     
     private final CompletableFuture completionFuture;

From 4aac24aa544483e0d114f70524a9e7fba173d327 Mon Sep 17 00:00:00 2001
From: Bennett Lynch 
Date: Wed, 3 Nov 2021 20:31:17 -0700
Subject: [PATCH 3/4] Add additional test coverage

---
 .../CompletedDirectoryUploadTest.java         |   3 +-
 .../internal/DefaultDirectoryUploadTest.java  |  31 ++++
 .../s3/internal/DefaultDownloadTest.java      |  30 ++++
 .../s3/internal/DefaultFileDownloadTest.java  |  30 ++++
 .../s3/internal/DefaultFileUploadTest.java    |  29 ++++
 .../s3/internal/DefaultUploadTest.java        |  29 ++++
 .../S3TransferManagerListenerTest.java        |  97 +++++++++++-
 .../s3/internal/S3TransferManagerTest.java    | 141 +++++++++++++-----
 8 files changed, 350 insertions(+), 40 deletions(-)
 rename services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/{internal => }/CompletedDirectoryUploadTest.java (88%)
 create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
 create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownloadTest.java
 create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownloadTest.java
 create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUploadTest.java
 create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultUploadTest.java

diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUploadTest.java
similarity index 88%
rename from services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java
rename to services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUploadTest.java
index bfe9dbd438c7..4d18abc06ce6 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedDirectoryUploadTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDirectoryUploadTest.java
@@ -13,11 +13,10 @@
  * permissions and limitations under the License.
  */
 
-package software.amazon.awssdk.transfer.s3.internal;
+package software.amazon.awssdk.transfer.s3;
 
 import nl.jqno.equalsverifier.EqualsVerifier;
 import org.junit.Test;
-import software.amazon.awssdk.transfer.s3.CompletedDirectoryUpload;
 
 public class CompletedDirectoryUploadTest {
 
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
new file mode 100644
index 000000000000..c230c5309be4
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+import software.amazon.awssdk.transfer.s3.UploadRequest;
+
+public class DefaultDirectoryUploadTest {
+
+    @Test
+    public void equals_hashcode() {
+        EqualsVerifier.forClass(UploadRequest.class)
+                      .withNonnullFields("completionFuture")
+                      .verify();
+    }
+
+}
\ No newline at end of file
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownloadTest.java
new file mode 100644
index 000000000000..4d5889e9bb2a
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownloadTest.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class DefaultDownloadTest {
+
+    @Test
+    public void equals_hashcode() {
+        EqualsVerifier.forClass(DefaultDownload.class)
+                      .withNonnullFields("completionFuture", "progress")
+                      .verify();
+    }
+
+}
\ No newline at end of file
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownloadTest.java
new file mode 100644
index 000000000000..a07cb6a26cb1
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileDownloadTest.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class DefaultFileDownloadTest {
+
+    @Test
+    public void equals_hashcode() {
+        EqualsVerifier.forClass(DefaultFileDownload.class)
+                      .withNonnullFields("completionFuture", "progress")
+                      .verify();
+    }
+
+}
\ No newline at end of file
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUploadTest.java
new file mode 100644
index 000000000000..c0f36785482d
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultFileUploadTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class DefaultFileUploadTest {
+
+    @Test
+    public void equals_hashcode() {
+        EqualsVerifier.forClass(DefaultFileUpload.class)
+                      .withNonnullFields("completionFuture", "progress")
+                      .verify();
+    }
+}
\ No newline at end of file
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultUploadTest.java
new file mode 100644
index 000000000000..b08cc523dcb3
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultUploadTest.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class DefaultUploadTest {
+
+    @Test
+    public void equals_hashcode() {
+        EqualsVerifier.forClass(DefaultUpload.class)
+                      .withNonnullFields("completionFuture", "progress")
+                      .verify();
+    }
+}
\ No newline at end of file
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java
index 8a08e422c438..716e03b94560 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java
@@ -42,6 +42,7 @@
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.stubbing.Answer;
+import software.amazon.awssdk.core.ResponseBytes;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.async.DrainingSubscriber;
@@ -50,11 +51,15 @@
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.transfer.s3.CompletedFileUpload;
+import software.amazon.awssdk.transfer.s3.Download;
 import software.amazon.awssdk.transfer.s3.DownloadFileRequest;
+import software.amazon.awssdk.transfer.s3.DownloadRequest;
 import software.amazon.awssdk.transfer.s3.FileDownload;
 import software.amazon.awssdk.transfer.s3.FileUpload;
 import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.Upload;
 import software.amazon.awssdk.transfer.s3.UploadFileRequest;
+import software.amazon.awssdk.transfer.s3.UploadRequest;
 import software.amazon.awssdk.transfer.s3.progress.TransferListener;
 
 public class S3TransferManagerListenerTest {
@@ -80,7 +85,7 @@ public void methodTeardown() {
     }
 
     @Test
-    public void upload_success_shouldInvokeListener() throws Exception {
+    public void uploadFile_success_shouldInvokeListener() throws Exception {
         TransferListener listener = mock(TransferListener.class);
 
         Path path = newTempFile();
@@ -124,7 +129,48 @@ public void upload_success_shouldInvokeListener() throws Exception {
     }
 
     @Test
-    public void download_success_shouldInvokeListener() throws Exception {
+    public void upload_success_shouldInvokeListener() throws Exception {
+        TransferListener listener = mock(TransferListener.class);
+
+        UploadRequest uploadRequest = UploadRequest.builder()
+                                                   .putObjectRequest(r -> r.bucket("bucket")
+                                                                           .key("key"))
+                                                   .requestBody(AsyncRequestBody.fromString("foo"))
+                                                   .overrideConfiguration(b -> b.addListener(listener))
+                                                   .build();
+        Upload upload = tm.upload(uploadRequest);
+
+        ArgumentCaptor captor1 =
+            ArgumentCaptor.forClass(TransferListener.Context.TransferInitiated.class);
+        verify(listener, timeout(1000).times(1)).transferInitiated(captor1.capture());
+        TransferListener.Context.TransferInitiated ctx1 = captor1.getValue();
+        assertThat(ctx1.request()).isSameAs(uploadRequest);
+        assertThat(ctx1.progressSnapshot().transferSizeInBytes()).hasValue(3L);
+        assertThat(ctx1.progressSnapshot().bytesTransferred()).isZero();
+
+        ArgumentCaptor captor2 =
+            ArgumentCaptor.forClass(TransferListener.Context.BytesTransferred.class);
+        verify(listener, timeout(1000).times(1)).bytesTransferred(captor2.capture());
+        TransferListener.Context.BytesTransferred ctx2 = captor2.getValue();
+        assertThat(ctx2.request()).isSameAs(uploadRequest);
+        assertThat(ctx2.progressSnapshot().transferSizeInBytes()).hasValue(3L);
+        assertThat(ctx2.progressSnapshot().bytesTransferred()).isPositive();
+
+        ArgumentCaptor captor3 =
+            ArgumentCaptor.forClass(TransferListener.Context.TransferComplete.class);
+        verify(listener, timeout(1000).times(1)).transferComplete(captor3.capture());
+        TransferListener.Context.TransferComplete ctx3 = captor3.getValue();
+        assertThat(ctx3.request()).isSameAs(uploadRequest);
+        assertThat(ctx3.progressSnapshot().transferSizeInBytes()).hasValue(3L);
+        assertThat(ctx3.progressSnapshot().bytesTransferred()).isEqualTo(3L);
+        assertThat(ctx3.completedTransfer()).isSameAs(upload.completionFuture().get());
+
+        upload.completionFuture().join();
+        verifyNoMoreInteractions(listener);
+    }
+
+    @Test
+    public void downloadFile_success_shouldInvokeListener() throws Exception {
         TransferListener listener = mock(TransferListener.class);
 
         DownloadFileRequest downloadRequest = DownloadFileRequest.builder()
@@ -167,7 +213,52 @@ public void download_success_shouldInvokeListener() throws Exception {
     }
 
     @Test
-    public void upload_failure_shouldInvokeListener() throws Exception {
+    public void download_success_shouldInvokeListener() throws Exception {
+        TransferListener listener = mock(TransferListener.class);
+
+        DownloadRequest> downloadRequest =
+            DownloadRequest.builder()
+                           .getObjectRequest(r -> r.bucket(
+                                                       "bucket")
+                                                   .key("key"))
+                           .responseTransformer(AsyncResponseTransformer.toBytes())
+                           .overrideConfiguration(b -> b.addListener(listener))
+                           .build();
+        Download> download = tm.download(downloadRequest);
+
+        ArgumentCaptor captor1 =
+            ArgumentCaptor.forClass(TransferListener.Context.TransferInitiated.class);
+        verify(listener, timeout(1000).times(1)).transferInitiated(captor1.capture());
+        TransferListener.Context.TransferInitiated ctx1 = captor1.getValue();
+        assertThat(ctx1.request()).isSameAs(downloadRequest);
+        // transferSize is not known until we receive GetObjectResponse header
+        assertThat(ctx1.progressSnapshot().transferSizeInBytes()).isNotPresent();
+        assertThat(ctx1.progressSnapshot().bytesTransferred()).isZero();
+
+        ArgumentCaptor captor2 =
+            ArgumentCaptor.forClass(TransferListener.Context.BytesTransferred.class);
+        verify(listener, timeout(1000).times(1)).bytesTransferred(captor2.capture());
+        TransferListener.Context.BytesTransferred ctx2 = captor2.getValue();
+        assertThat(ctx2.request()).isSameAs(downloadRequest);
+        // transferSize should now be known
+        assertThat(ctx2.progressSnapshot().transferSizeInBytes()).hasValue(contentLength);
+        assertThat(ctx2.progressSnapshot().bytesTransferred()).isPositive();
+
+        ArgumentCaptor captor3 =
+            ArgumentCaptor.forClass(TransferListener.Context.TransferComplete.class);
+        verify(listener, timeout(1000).times(1)).transferComplete(captor3.capture());
+        TransferListener.Context.TransferComplete ctx3 = captor3.getValue();
+        assertThat(ctx3.request()).isSameAs(downloadRequest);
+        assertThat(ctx3.progressSnapshot().transferSizeInBytes()).hasValue(contentLength);
+        assertThat(ctx3.progressSnapshot().bytesTransferred()).isEqualTo(contentLength);
+        assertThat(ctx3.completedTransfer()).isSameAs(download.completionFuture().get());
+
+        download.completionFuture().join();
+        verifyNoMoreInteractions(listener);
+    }
+
+    @Test
+    public void uploadFile_failure_shouldInvokeListener() throws Exception {
         TransferListener listener = mock(TransferListener.class);
 
         Path path = newTempFile();
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
index 79659cb1459f..86c61bcedf78 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
@@ -27,14 +27,17 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import software.amazon.awssdk.core.ResponseBytes;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.transfer.s3.CompletedDownload;
 import software.amazon.awssdk.transfer.s3.CompletedFileDownload;
 import software.amazon.awssdk.transfer.s3.CompletedFileUpload;
+import software.amazon.awssdk.transfer.s3.CompletedUpload;
 import software.amazon.awssdk.transfer.s3.DownloadFileRequest;
 import software.amazon.awssdk.transfer.s3.S3TransferManager;
 import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
@@ -66,16 +69,14 @@ public void defaultTransferManager_shouldNotThrowException() {
     }
 
     @Test
-    public void upload_returnsResponse() {
+    public void uploadFile_returnsResponse() {
         PutObjectResponse response = PutObjectResponse.builder().build();
         when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
-                .thenReturn(CompletableFuture.completedFuture(response));
+            .thenReturn(CompletableFuture.completedFuture(response));
 
-        CompletedFileUpload completedFileUpload = tm.uploadFile(UploadFileRequest.builder()
-                                                                                 .putObjectRequest(r -> r.bucket("bucket")
-                                                                                         .key("key"))
-                                                                                 .source(Paths.get("."))
-                                                                                 .build())
+        CompletedFileUpload completedFileUpload = tm.uploadFile(u -> u.putObjectRequest(p -> p.bucket("bucket")
+                                                                                              .key("key"))
+                                                                      .source(Paths.get(".")))
                                                     .completionFuture()
                                                     .join();
 
@@ -83,16 +84,29 @@ public void upload_returnsResponse() {
     }
 
     @Test
-    public void upload_cancel_shouldForwardCancellation() {
+    public void upload_returnsResponse() {
+        PutObjectResponse response = PutObjectResponse.builder().build();
+        when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
+            .thenReturn(CompletableFuture.completedFuture(response));
+
+        CompletedUpload completedUpload = tm.upload(u -> u.putObjectRequest(p -> p.bucket("bucket")
+                                                                                  .key("key"))
+                                                          .requestBody(AsyncRequestBody.fromString("foo")))
+                                            .completionFuture()
+                                            .join();
+
+        assertThat(completedUpload.response()).isEqualTo(response);
+    }
+
+    @Test
+    public void uploadFile_cancel_shouldForwardCancellation() {
         CompletableFuture s3CrtFuture = new CompletableFuture<>();
         when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
             .thenReturn(s3CrtFuture);
 
-        CompletableFuture future = tm.uploadFile(UploadFileRequest.builder()
-                                                                                       .putObjectRequest(r -> r.bucket("bucket")
-                                                                                         .key("key"))
-                                                                                       .source(Paths.get("."))
-                                                                                       .build())
+        CompletableFuture future = tm.uploadFile(u -> u.putObjectRequest(p -> p.bucket("bucket")
+                                                                                                    .key("key"))
+                                                                            .source(Paths.get(".")))
                                                           .completionFuture();
 
         future.cancel(true);
@@ -100,66 +114,123 @@ public void upload_cancel_shouldForwardCancellation() {
     }
 
     @Test
-    public void download_returnsResponse() {
+    public void upload_cancel_shouldForwardCancellation() {
+        CompletableFuture s3CrtFuture = new CompletableFuture<>();
+        when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)))
+            .thenReturn(s3CrtFuture);
+
+        CompletableFuture future = tm.upload(u -> u.putObjectRequest(p -> p.bucket("bucket")
+                                                                                            .key("key"))
+                                                                    .requestBody(AsyncRequestBody.fromString("foo")))
+                                                      .completionFuture();
+
+        future.cancel(true);
+        assertThat(s3CrtFuture).isCancelled();
+    }
+
+    @Test
+    public void downloadFile_returnsResponse() {
         GetObjectResponse response = GetObjectResponse.builder().build();
         when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)))
             .thenReturn(CompletableFuture.completedFuture(response));
 
-        CompletedFileDownload completedFileDownload = tm.downloadFile(DownloadFileRequest.builder()
-                                                                                         .getObjectRequest(r -> r.bucket("bucket")
-                                                                                                 .key("key"))
-                                                                                         .destination(Paths.get("."))
-                                                                                         .build())
+        CompletedFileDownload completedFileDownload = tm.downloadFile(d -> d.getObjectRequest(g -> g.bucket("bucket")
+                                                                                                    .key("key"))
+                                                                            .destination(Paths.get(".")))
                                                         .completionFuture()
                                                         .join();
         assertThat(completedFileDownload.response()).isEqualTo(response);
     }
 
     @Test
-    public void download_cancel_shouldForwardCancellation() {
+    public void downloadFile_cancel_shouldForwardCancellation() {
         CompletableFuture s3CrtFuture = new CompletableFuture<>();
         when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)))
             .thenReturn(s3CrtFuture);
 
-        CompletableFuture future = tm.downloadFile(DownloadFileRequest.builder()
-                                                                                             .getObjectRequest(r -> r.bucket("bucket")
-                                                                                                         .key("key"))
-                                                                                             .destination(Paths.get("."))
-                                                                                             .build())
+        CompletableFuture future = tm.downloadFile(d -> d
+                                                                .getObjectRequest(g -> g.bucket(
+                                                                                            "bucket")
+                                                                                        .key("key"))
+                                                                .destination(Paths.get(".")))
                                                             .completionFuture();
         future.cancel(true);
         assertThat(s3CrtFuture).isCancelled();
     }
 
+    @Test
+    public void download_cancel_shouldForwardCancellation() {
+        CompletableFuture s3CrtFuture = new CompletableFuture<>();
+        when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)))
+            .thenReturn(s3CrtFuture);
+
+        CompletableFuture>> future =
+            tm.download(d -> d
+                  .getObjectRequest(g -> g.bucket("bucket")
+                                          .key("key"))
+                  .responseTransformer(AsyncResponseTransformer.toBytes()))
+              .completionFuture();
+        future.cancel(true);
+        assertThat(s3CrtFuture).isCancelled();
+    }
+
     @Test
     public void objectLambdaArnBucketProvided_shouldThrowException() {
         String objectLambdaArn = "arn:xxx:s3-object-lambda";
-        assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn)
-                                                                             .key("key")).source(Paths.get(".")))
+
+        assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn).key("key"))
+                                                     .source(Paths.get(".")))
+                                   .completionFuture().join())
+            .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
+
+        assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn).key("key"))
+                                                 .requestBody(AsyncRequestBody.fromString("foo")))
+                                   .completionFuture().join())
+            .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
+
+        assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn).key("key"))
+                                                       .destination(Paths.get(".")))
                                    .completionFuture().join())
             .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
 
-        assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn)
-                                                                               .key("key")).destination(Paths.get("."))).completionFuture().join())
+        assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn).key("key"))
+                                                   .responseTransformer(AsyncResponseTransformer.toBytes()))
+                                   .completionFuture().join())
             .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
 
-        assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(objectLambdaArn).sourceDirectory(Paths.get("."))).completionFuture().join())
+        assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(objectLambdaArn)
+                                                          .sourceDirectory(Paths.get(".")))
+                                   .completionFuture().join())
             .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
     }
 
     @Test
     public void mrapArnProvided_shouldThrowException() {
         String mrapArn = "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap";
-        assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(mrapArn)
-                                                                             .key("key")).source(Paths.get(".")))
+
+        assertThatThrownBy(() -> tm.uploadFile(b -> b.putObjectRequest(p -> p.bucket(mrapArn).key("key"))
+                                                     .source(Paths.get(".")))
                                    .completionFuture().join())
             .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
 
-        assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(mrapArn)
-                                                                               .key("key")).destination(Paths.get("."))).completionFuture().join())
+        assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(mrapArn).key("key"))
+                                                 .requestBody(AsyncRequestBody.fromString("foo")))
+                                   .completionFuture().join())
             .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
 
-        assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(mrapArn).sourceDirectory(Paths.get("."))).completionFuture().join())
+        assertThatThrownBy(() -> tm.downloadFile(b -> b.getObjectRequest(p -> p.bucket(mrapArn).key("key"))
+                                                       .destination(Paths.get(".")))
+                                   .completionFuture().join())
+            .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
+
+        assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(mrapArn).key("key"))
+                                                   .responseTransformer(AsyncResponseTransformer.toBytes()))
+                                   .completionFuture().join())
+            .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
+
+        assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(mrapArn)
+                                                          .sourceDirectory(Paths.get(".")))
+                                   .completionFuture().join())
             .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
     }
 

From 9f8e42b08a34cc32d7ba7ac92b340be8153de50c Mon Sep 17 00:00:00 2001
From: Bennett Lynch 
Date: Wed, 3 Nov 2021 21:44:20 -0700
Subject: [PATCH 4/4] Fix equals/hashCode unit test

---
 .../awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
index c230c5309be4..2690a2fdb14c 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultDirectoryUploadTest.java
@@ -24,7 +24,7 @@ public class DefaultDirectoryUploadTest {
     @Test
     public void equals_hashcode() {
         EqualsVerifier.forClass(UploadRequest.class)
-                      .withNonnullFields("completionFuture")
+                      .withNonnullFields("putObjectRequest", "requestBody")
                       .verify();
     }