co
return this;
}
+ /**
+ * Configuration settings for how {@link S3TransferManager} should process the request. The
+ * {@link S3TransferManager} already provides sensible defaults. All values are optional.
+ *
+ * @param transferConfiguration the configuration to use
+ * @return Returns a reference to this object so that method calls can be chained together.
+ * @see #transferConfiguration(Consumer)
+ */
+ Builder transferConfiguration(S3TransferManagerOverrideConfiguration transferConfiguration);
+
+ /**
+ * Configuration settings for how {@link S3TransferManager} should process the request. The
+ * {@link S3TransferManager} already provides sensible defaults. All values are optional.
+ *
+ *
+ * This is a convenience method that creates an instance of the {@link S3TransferManagerOverrideConfiguration} builder
+ * avoiding the need to create one manually via {@link S3TransferManagerOverrideConfiguration#builder()}.
+ *
+ * @param configuration the configuration to use
+ * @return Returns a reference to this object so that method calls can be chained together.
+ * @see #transferConfiguration(S3TransferManagerOverrideConfiguration)
+ */
+ default Builder transferConfiguration(Consumer configuration) {
+ Validate.paramNotNull(configuration, "configuration");
+ S3TransferManagerOverrideConfiguration.Builder builder = S3TransferManagerOverrideConfiguration.builder();
+ configuration.accept(builder);
+ transferConfiguration(builder.build());
+ return this;
+ }
+
/**
* Build an instance of {@link S3TransferManager} based on the settings supplied to this builder
*
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfiguration.java
new file mode 100644
index 000000000000..4986973cdb14
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfiguration.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.Validate;
+import software.amazon.awssdk.utils.builder.CopyableBuilder;
+import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
+
+/**
+ * Configuration options for how {@link S3TransferManager} processes requests.
+ *
+ * All values are optional, and not specifying them will use default values provided bt the SDK.
+ *
+ *
Use {@link #builder()} to create a set of options.
+ * @see S3TransferManager.Builder#transferConfiguration(S3TransferManagerOverrideConfiguration)
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public final class S3TransferManagerOverrideConfiguration implements
+ ToCopyableBuilder {
+ private final Executor executor;
+ private final UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration;
+
+ private S3TransferManagerOverrideConfiguration(DefaultBuilder builder) {
+ this.executor = builder.executor;
+ this.uploadDirectoryConfiguration = builder.uploadDirectoryConfiguration;
+ }
+
+ /**
+ * @return the optional SDK executor specified
+ */
+ public Optional executor() {
+ return Optional.ofNullable(executor);
+ }
+
+ /**
+ * @return the optional upload directory configuration specified
+ */
+ public Optional uploadDirectoryConfiguration() {
+ return Optional.ofNullable(uploadDirectoryConfiguration);
+ }
+
+ @Override
+ public Builder toBuilder() {
+ return new DefaultBuilder(this);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ S3TransferManagerOverrideConfiguration that = (S3TransferManagerOverrideConfiguration) o;
+
+ if (!Objects.equals(executor, that.executor)) {
+ return false;
+ }
+ return Objects.equals(uploadDirectoryConfiguration, that.uploadDirectoryConfiguration);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = executor != null ? executor.hashCode() : 0;
+ result = 31 * result + (uploadDirectoryConfiguration != null ? uploadDirectoryConfiguration.hashCode() : 0);
+ return result;
+ }
+
+ /**
+ * Creates a default builder for {@link S3TransferManagerOverrideConfiguration}.
+ */
+ public static Builder builder() {
+ return new DefaultBuilder();
+ }
+
+ public static Class serializableBuilderClass() {
+ return DefaultBuilder.class;
+ }
+
+ /**
+ * The builder definition for a {@link S3TransferManagerOverrideConfiguration}.
+ */
+ public interface Builder extends CopyableBuilder {
+
+ /**
+ * Specify the executor that {@link S3TransferManager} will use to execute background tasks before handing them off to
+ * the underlying S3 async client, such as visiting file tree in a
+ * {@link S3TransferManager#uploadDirectory(UploadDirectoryRequest)} operation
+ *
+ *
+ * The SDK will create an executor if not provided
+ *
+ *
+ * This executor must be shut down by the user when it is ready to be disposed. The SDK will not close the executor
+ * when the s3 transfer manager is closed.
+ *
+ * @param executor the executor to use
+ * @return this builder for method chaining.
+ */
+ Builder executor(Executor executor);
+
+ /**
+ * Specify the configuration options for upload directory operation
+ *
+ * @param uploadDirectoryConfiguration the configuration for upload directory
+ * @return this builder for method chaining.
+ * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest)
+ */
+ Builder uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration);
+
+ /**
+ * Similar to {@link #uploadDirectoryConfiguration}, but takes a lambda to configure a new
+ * {@link UploadDirectoryOverrideConfiguration.Builder}. This removes the need to call
+ * {@link UploadDirectoryOverrideConfiguration#builder()} and
+ * {@link UploadDirectoryOverrideConfiguration.Builder#build()}.
+ *
+ * @param uploadConfigurationBuilder the configuration for upload directory
+ * @return this builder for method chaining.
+ * @see #uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration)
+ */
+ default Builder uploadDirectoryConfiguration(Consumer
+ uploadConfigurationBuilder) {
+ Validate.paramNotNull(uploadConfigurationBuilder, "uploadConfigurationBuilder");
+ return uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration.builder()
+ .applyMutation(uploadConfigurationBuilder)
+ .build());
+ }
+ }
+
+ private static final class DefaultBuilder implements Builder {
+ private Executor executor;
+ private UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration;
+
+ private DefaultBuilder() {
+ }
+
+ private DefaultBuilder(S3TransferManagerOverrideConfiguration configuration) {
+ this.executor = configuration.executor;
+ this.uploadDirectoryConfiguration = configuration.uploadDirectoryConfiguration;
+ }
+
+ @Override
+ public Builder executor(Executor executor) {
+ this.executor = executor;
+ return this;
+ }
+
+ public void setExecutor(Executor executor) {
+ executor(executor);
+ }
+
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ @Override
+ public Builder uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration) {
+ this.uploadDirectoryConfiguration = uploadDirectoryConfiguration;
+ return this;
+ }
+
+ public void setUploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration) {
+ uploadDirectoryConfiguration(uploadDirectoryConfiguration);
+ }
+
+ public UploadDirectoryOverrideConfiguration getUploadDirectoryConfiguration() {
+ return uploadDirectoryConfiguration;
+ }
+
+ @Override
+ public S3TransferManagerOverrideConfiguration build() {
+ return new S3TransferManagerOverrideConfiguration(this);
+ }
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfiguration.java
new file mode 100644
index 000000000000..a4bf7135ebb1
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfiguration.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import java.util.Objects;
+import java.util.Optional;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.ToString;
+import software.amazon.awssdk.utils.Validate;
+import software.amazon.awssdk.utils.builder.CopyableBuilder;
+import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
+
+/**
+ * Configuration options for {@link S3TransferManager#uploadDirectory}. All values are optional, and not specifying them will
+ * use the SDK default values.
+ *
+ * Use {@link #builder()} to create a set of options.
+ * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest)
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public final class UploadDirectoryOverrideConfiguration implements ToCopyableBuilder {
+
+ private final Boolean followSymbolicLinks;
+ private final Integer maxDepth;
+ private final Boolean recursive;
+
+ public UploadDirectoryOverrideConfiguration(DefaultBuilder builder) {
+ this.followSymbolicLinks = builder.followSymbolicLinks;
+ this.maxDepth = Validate.isPositiveOrNull(builder.maxDepth, "maxDepth");
+ this.recursive = builder.recursive;
+ }
+
+ /**
+ * @return whether to follow symbolic links
+ * @see Builder#followSymbolicLinks(Boolean)
+ */
+ public Optional followSymbolicLinks() {
+ return Optional.ofNullable(followSymbolicLinks);
+ }
+
+ /**
+ * @return the maximum number of directory levels to traverse
+ * @see Builder#maxDepth(Integer)
+ */
+ public Optional maxDepth() {
+ return Optional.ofNullable(maxDepth);
+ }
+
+ /**
+ * @return whether to recursively upload all files under the specified directory
+ * @see Builder#recursive(Boolean)
+ */
+ public Optional recursive() {
+ return Optional.ofNullable(recursive);
+ }
+
+ @Override
+ public Builder toBuilder() {
+ return new DefaultBuilder(this);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ UploadDirectoryOverrideConfiguration that = (UploadDirectoryOverrideConfiguration) o;
+
+ if (!Objects.equals(followSymbolicLinks, that.followSymbolicLinks)) {
+ return false;
+ }
+ if (!Objects.equals(maxDepth, that.maxDepth)) {
+ return false;
+ }
+ return Objects.equals(recursive, that.recursive);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = followSymbolicLinks != null ? followSymbolicLinks.hashCode() : 0;
+ result = 31 * result + (maxDepth != null ? maxDepth.hashCode() : 0);
+ result = 31 * result + (recursive != null ? recursive.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return ToString.builder("UploadDirectoryConfiguration")
+ .add("followSymbolicLinks", followSymbolicLinks)
+ .add("maxDepth", maxDepth)
+ .add("recursive", recursive)
+ .build();
+ }
+
+ public static Builder builder() {
+ return new DefaultBuilder();
+ }
+
+ public static Class serializableBuilderClass() {
+ return DefaultBuilder.class;
+ }
+
+ // TODO: consider consolidating maxDepth and recursive
+ public interface Builder extends CopyableBuilder {
+
+ /**
+ * Specify whether to recursively upload all files under the specified directory
+ *
+ *
+ * Default to true
+ *
+ * @param recursive whether enable recursive upload
+ * @return This builder for method chaining.
+ */
+ Builder recursive(Boolean recursive);
+
+ /**
+ * Specify whether to follow symbolic links when traversing the file tree.
+ *
+ * Default to false
+ *
+ * @param followSymbolicLinks whether to follow symbolic links
+ * @return This builder for method chaining.
+ */
+ Builder followSymbolicLinks(Boolean followSymbolicLinks);
+
+ /**
+ * Specify the maximum number of levels of directories to visit. Must be positive.
+ * 1 means only the files directly within the provided source directory are visited.
+ *
+ *
+ * Default to {@code Integer.MAX_VALUE}
+ *
+ * @param maxDepth the maximum number of directory levels to visit
+ * @return This builder for method chaining.
+ */
+ Builder maxDepth(Integer maxDepth);
+
+ @Override
+ UploadDirectoryOverrideConfiguration build();
+ }
+
+ private static final class DefaultBuilder implements Builder {
+ private Boolean followSymbolicLinks;
+ private Integer maxDepth;
+ private Boolean recursive;
+
+ private DefaultBuilder(UploadDirectoryOverrideConfiguration configuration) {
+ this.followSymbolicLinks = configuration.followSymbolicLinks;
+ this.maxDepth = configuration.maxDepth;
+ this.recursive = configuration.recursive;
+ }
+
+ private DefaultBuilder() {
+ }
+
+ @Override
+ public Builder recursive(Boolean recursive) {
+ this.recursive = recursive;
+ return this;
+ }
+
+ public Boolean getRecursive() {
+ return recursive;
+ }
+
+ public void setRecursive(Boolean recursive) {
+ recursive(recursive);
+ }
+
+ @Override
+ public Builder followSymbolicLinks(Boolean followSymbolicLinks) {
+ this.followSymbolicLinks = followSymbolicLinks;
+ return this;
+ }
+
+ public void setFollowSymbolicLinks(Boolean followSymbolicLinks) {
+ followSymbolicLinks(followSymbolicLinks);
+ }
+
+ public Boolean getFollowSymbolicLinks() {
+ return followSymbolicLinks;
+ }
+
+ @Override
+ public Builder maxDepth(Integer maxDepth) {
+ this.maxDepth = maxDepth;
+ return this;
+ }
+
+ public void setMaxDepth(Integer maxDepth) {
+ maxDepth(maxDepth);
+ }
+
+ public Integer getMaxDepth() {
+ return maxDepth;
+ }
+
+ @Override
+ public UploadDirectoryOverrideConfiguration build() {
+ return new UploadDirectoryOverrideConfiguration(this);
+ }
+
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java
new file mode 100644
index 000000000000..fcf9cbb1c46c
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequest.java
@@ -0,0 +1,345 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+
+import java.nio.file.Path;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Consumer;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.Validate;
+import software.amazon.awssdk.utils.builder.CopyableBuilder;
+import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
+
+/**
+ * Request object to upload a local directory to S3 using the Transfer Manager.
+ *
+ * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest)
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public final class UploadDirectoryRequest implements TransferRequest, ToCopyableBuilder {
+
+ private final Path sourceDirectory;
+ private final String bucket;
+ private final String prefix;
+ private final UploadDirectoryOverrideConfiguration overrideConfiguration;
+ private final String delimiter;
+
+ public UploadDirectoryRequest(DefaultBuilder builder) {
+ this.sourceDirectory = Validate.paramNotNull(builder.sourceDirectory, "sourceDirectory");
+ this.bucket = Validate.paramNotNull(builder.bucket, "bucket");
+ this.prefix = builder.prefix;
+ this.overrideConfiguration = builder.configuration;
+ this.delimiter = builder.delimiter;
+ }
+
+ /**
+ * The source directory to upload
+ *
+ * @return the source directory
+ * @see Builder#sourceDirectory(Path)
+ */
+ public Path sourceDirectory() {
+ return sourceDirectory;
+ }
+
+ /**
+ * The name of the bucket to upload objects to.
+ *
+ * @return bucket name
+ * @see Builder#bucket(String)
+ */
+ public String bucket() {
+ return bucket;
+ }
+
+ /**
+ * @return the optional key prefix
+ * @see Builder#prefix(String)
+ */
+ public Optional prefix() {
+ return Optional.ofNullable(prefix);
+ }
+
+ /**
+ * @return the optional delimiter
+ * @see Builder#delimiter(String)
+ */
+ public Optional delimiter() {
+ return Optional.ofNullable(delimiter);
+ }
+
+ /**
+ * @return the optional override configuration
+ * @see Builder#overrideConfiguration(UploadDirectoryOverrideConfiguration)
+ */
+ public Optional overrideConfiguration() {
+ return Optional.ofNullable(overrideConfiguration);
+ }
+
+ public static Builder builder() {
+ return new DefaultBuilder();
+ }
+
+ public static Class serializableBuilderClass() {
+ return DefaultBuilder.class;
+ }
+
+ @Override
+ public Builder toBuilder() {
+ return new DefaultBuilder(this);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ UploadDirectoryRequest that = (UploadDirectoryRequest) o;
+
+ if (!sourceDirectory.equals(that.sourceDirectory)) {
+ return false;
+ }
+ if (!bucket.equals(that.bucket)) {
+ return false;
+ }
+ if (!Objects.equals(prefix, that.prefix)) {
+ return false;
+ }
+ if (!Objects.equals(delimiter, that.delimiter)) {
+ return false;
+ }
+ return Objects.equals(overrideConfiguration, that.overrideConfiguration);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = sourceDirectory.hashCode();
+ result = 31 * result + bucket.hashCode();
+ result = 31 * result + (prefix != null ? prefix.hashCode() : 0);
+ result = 31 * result + (delimiter != null ? delimiter.hashCode() : 0);
+ result = 31 * result + (overrideConfiguration != null ? overrideConfiguration.hashCode() : 0);
+ return result;
+ }
+
+ public interface Builder extends CopyableBuilder {
+
+ /**
+ * Specify the source directory to upload. The source directory must exist.
+ * Fle wildcards are not supported and treated literally. Hidden files/directories are visited.
+ *
+ *
+ * Note that the current user must have read access to all directories and files,
+ * otherwise {@link SecurityException} will be thrown.
+ *
+ * @param sourceDirectory the source directory
+ * @return This builder for method chaining.
+ * @see UploadDirectoryOverrideConfiguration
+ */
+ Builder sourceDirectory(Path sourceDirectory);
+
+ /**
+ * The name of the bucket to upload objects to.
+ *
+ * @param bucket the bucket name
+ * @return This builder for method chaining.
+ */
+ Builder bucket(String bucket);
+
+ /**
+ * Specify the key prefix to use for the objects. If not provided, files will be uploaded to the root of the bucket
+ *
+ * See Organizing objects using
+ * prefixes
+ *
+ *
+ * Note: if the provided prefix ends with the same string as delimiter, it will get "normalized" when generating the key
+ * name. For example, assuming the prefix provided is "foo/" and the delimiter is "/" and the source directory has the
+ * following structure:
+ *
+ *
+ * |- test
+ * |- obj1.txt
+ * |- obj2.txt
+ *
+ *
+ * the object keys will be "foo/obj1.txt" and "foo/obj2.txt" as apposed to "foo//obj1.txt" and "foo//obj2.txt"
+ *
+ * @param prefix the key prefix
+ * @return This builder for method chaining.
+ * @see #delimiter(String)
+ */
+ Builder prefix(String prefix);
+
+ /**
+ * Specify the delimiter. A delimiter causes a list operation to roll up all the keys that share a common prefix into a
+ * single summary list result. If not provided, {@code "/"} will be used.
+ *
+ * See Organizing objects using
+ * prefixes
+ *
+ *
+ * Note: if the provided prefix ends with the same string as delimiter, it will get "normalized" when generating the key
+ * name. For example, assuming the prefix provided is "foo/" and the delimiter is "/" and the source directory has the
+ * following structure:
+ *
+ *
+ * |- test
+ * |- obj1.txt
+ * |- obj2.txt
+ *
+ *
+ * the object keys will be "foo/obj1.txt" and "foo/obj2.txt" as apposed to "foo//obj1.txt" and "foo//obj2.txt"
+ *
+ * @param delimiter the delimiter
+ * @return This builder for method chaining.
+ * @see #prefix(String)
+ */
+ Builder delimiter(String delimiter);
+
+ /**
+ * Add an optional request override configuration.
+ *
+ * @param configuration The override configuration.
+ * @return This builder for method chaining.
+ */
+ Builder overrideConfiguration(UploadDirectoryOverrideConfiguration configuration);
+
+ /**
+ * Similar to {@link #overrideConfiguration(UploadDirectoryOverrideConfiguration)}, but takes a lambda to configure a new
+ * {@link UploadDirectoryOverrideConfiguration.Builder}. This removes the need to call
+ * {@link UploadDirectoryOverrideConfiguration#builder()} and
+ * {@link UploadDirectoryOverrideConfiguration.Builder#build()}.
+ *
+ * @param uploadConfigurationBuilder the upload configuration
+ * @return this builder for method chaining.
+ * @see #overrideConfiguration(UploadDirectoryOverrideConfiguration)
+ */
+ default Builder overrideConfiguration(Consumer uploadConfigurationBuilder) {
+ Validate.paramNotNull(uploadConfigurationBuilder, "uploadConfigurationBuilder");
+ return overrideConfiguration(UploadDirectoryOverrideConfiguration.builder()
+ .applyMutation(uploadConfigurationBuilder)
+ .build());
+ }
+
+ @Override
+ UploadDirectoryRequest build();
+ }
+
+ private static final class DefaultBuilder implements Builder {
+
+ private Path sourceDirectory;
+ private String bucket;
+ private String prefix;
+ private UploadDirectoryOverrideConfiguration configuration;
+ private String delimiter;
+
+ private DefaultBuilder() {
+ }
+
+ private DefaultBuilder(UploadDirectoryRequest request) {
+ this.sourceDirectory = request.sourceDirectory;
+ this.bucket = request.bucket;
+ this.prefix = request.prefix;
+ this.configuration = request.overrideConfiguration;
+ }
+
+ @Override
+ public Builder sourceDirectory(Path sourceDirectory) {
+ this.sourceDirectory = sourceDirectory;
+ return this;
+ }
+
+ public void setSourceDirectory(Path sourceDirectory) {
+ sourceDirectory(sourceDirectory);
+ }
+
+ public Path getSourceDirectory() {
+ return sourceDirectory;
+ }
+
+ @Override
+ public Builder bucket(String bucket) {
+ this.bucket = bucket;
+ return this;
+ }
+
+ public void setBucket(String bucket) {
+ bucket(bucket);
+ }
+
+ public String getBucket() {
+ return bucket;
+ }
+
+ @Override
+ public Builder prefix(String prefix) {
+ this.prefix = prefix;
+ return this;
+ }
+
+ public void setPrefix(String prefix) {
+ prefix(prefix);
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ @Override
+ public Builder delimiter(String delimiter) {
+ this.delimiter = delimiter;
+ return this;
+ }
+
+ public void setDelimiter(String delimiter) {
+ delimiter(delimiter);
+ }
+
+ public String getDelimiter() {
+ return delimiter;
+ }
+
+ @Override
+ public Builder overrideConfiguration(UploadDirectoryOverrideConfiguration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ public void setOverrideConfiguration(UploadDirectoryOverrideConfiguration configuration) {
+ overrideConfiguration(configuration);
+ }
+
+ public UploadDirectoryOverrideConfiguration getOverrideConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public UploadDirectoryRequest build() {
+ return new UploadDirectoryRequest(this);
+ }
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java
new file mode 100644
index 000000000000..39ef2d7b6736
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectoryTransfer.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import java.util.concurrent.CompletableFuture;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+
+@SdkPublicApi
+@SdkPreviewApi
+public final class UploadDirectoryTransfer implements Transfer {
+ private final CompletableFuture completionFuture;
+
+ private UploadDirectoryTransfer(DefaultBuilder builder) {
+ this.completionFuture = builder.completionFuture;
+ }
+
+ @Override
+ public CompletableFuture completionFuture() {
+ return completionFuture;
+ }
+
+ public static Builder builder() {
+ return new DefaultBuilder();
+ }
+
+ public static Class serializableBuilderClass() {
+ return DefaultBuilder.class;
+ }
+
+ public interface Builder {
+
+ /**
+ * Specifies the future that will be completed when this transfer is complete.
+ *
+ * @param completionFuture the future that will be completed when this transfer is complete.
+ * @return This builder for method chaining.
+ */
+ Builder completionFuture(CompletableFuture completionFuture);
+
+ /**
+ * Builds a {@link UploadDirectoryTransfer} based on the properties supplied to this builder
+ *
+ * @return An initialized {@link UploadDirectoryTransfer}
+ */
+ UploadDirectoryTransfer build();
+ }
+
+ private static final class DefaultBuilder implements Builder {
+ private CompletableFuture completionFuture;
+
+ private DefaultBuilder() {
+ }
+
+ @Override
+ public DefaultBuilder completionFuture(CompletableFuture completionFuture) {
+ this.completionFuture = completionFuture;
+ return this;
+ }
+
+ public void setCompletionFuture(CompletableFuture completionFuture) {
+ completionFuture(completionFuture);
+ }
+
+ @Override
+ public UploadDirectoryTransfer build() {
+ return new UploadDirectoryTransfer(this);
+ }
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java
index ac230fe17ee3..d475f38f20f7 100644
--- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java
@@ -25,12 +25,14 @@
import software.amazon.awssdk.annotations.SdkPreviewApi;
import software.amazon.awssdk.annotations.SdkPublicApi;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.utils.ToString;
import software.amazon.awssdk.utils.Validate;
import software.amazon.awssdk.utils.builder.CopyableBuilder;
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
/**
* Upload an object to S3 using {@link S3TransferManager}.
+ * @see S3TransferManager#upload(UploadRequest)
*/
@SdkPublicApi
@SdkPreviewApi
@@ -68,6 +70,10 @@ public static Builder builder() {
return new BuilderImpl();
}
+ public static Class serializableBuilderClass() {
+ return BuilderImpl.class;
+ }
+
@Override
public Builder toBuilder() {
return new BuilderImpl();
@@ -97,6 +103,14 @@ public int hashCode() {
return result;
}
+ @Override
+ public String toString() {
+ return ToString.builder("UploadRequest")
+ .add("putObjectRequest", putObjectRequest)
+ .add("source", source)
+ .build();
+ }
+
/**
* A builder for a {@link UploadRequest}, created with {@link #builder()}
*/
@@ -170,12 +184,28 @@ public Builder source(Path source) {
return this;
}
+ public Path getSource() {
+ return source;
+ }
+
+ public void setSource(Path source) {
+ source(source);
+ }
+
@Override
public Builder putObjectRequest(PutObjectRequest putObjectRequest) {
this.putObjectRequest = putObjectRequest;
return this;
}
+ public PutObjectRequest getPutObjectRequest() {
+ return putObjectRequest;
+ }
+
+ public void setPutObjectRequest(PutObjectRequest putObjectRequest) {
+ putObjectRequest(putObjectRequest);
+ }
+
@Override
public UploadRequest build() {
return new UploadRequest(this);
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java
deleted file mode 100644
index 46a9716f61bd..000000000000
--- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/apache2.0
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.awssdk.transfer.s3.internal;
-
-import software.amazon.awssdk.annotations.SdkInternalApi;
-import software.amazon.awssdk.services.s3.model.GetObjectResponse;
-import software.amazon.awssdk.transfer.s3.CompletedDownload;
-
-@SdkInternalApi
-public final class DefaultCompletedDownload implements CompletedDownload {
- private final GetObjectResponse response;
-
- private DefaultCompletedDownload(Builder builder) {
- this.response = builder.response;
- }
-
- @Override
- public GetObjectResponse response() {
- return response;
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static final class Builder {
- private GetObjectResponse response;
-
- public Builder response(GetObjectResponse response) {
- this.response = response;
- return this;
- }
-
- public CompletedDownload build() {
- return new DefaultCompletedDownload(this);
- }
- }
-}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java
deleted file mode 100644
index 3d13b87c56bf..000000000000
--- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- * http://aws.amazon.com/apache2.0
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-package software.amazon.awssdk.transfer.s3.internal;
-
-import software.amazon.awssdk.annotations.SdkInternalApi;
-import software.amazon.awssdk.services.s3.model.PutObjectResponse;
-import software.amazon.awssdk.transfer.s3.CompletedUpload;
-
-@SdkInternalApi
-public final class DefaultCompletedUpload implements CompletedUpload {
- private final PutObjectResponse response;
-
- private DefaultCompletedUpload(BuilderImpl builder) {
- this.response = builder.response;
- }
-
- @Override
- public PutObjectResponse response() {
- return response;
- }
-
- /**
- * Creates a default builder for {@link CompletedUpload}.
- */
- public static Builder builder() {
- return new DefaultCompletedUpload.BuilderImpl();
- }
-
- interface Builder {
- Builder response(PutObjectResponse response);
-
- CompletedUpload build();
- }
-
- private static class BuilderImpl implements Builder {
- private PutObjectResponse response;
-
- @Override
- public Builder response(PutObjectResponse response) {
- this.response = response;
- return this;
- }
-
- @Override
- public CompletedUpload build() {
- return new DefaultCompletedUpload(this);
- }
- }
-}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java
index 5e34a995808d..4f33172bc1b4 100644
--- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java
@@ -18,8 +18,14 @@
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.annotations.SdkTestInternalApi;
+import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
+import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
+import software.amazon.awssdk.services.s3.internal.resource.S3AccessPointResource;
+import software.amazon.awssdk.services.s3.internal.resource.S3ArnConverter;
+import software.amazon.awssdk.services.s3.internal.resource.S3Resource;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
@@ -29,72 +35,164 @@
import software.amazon.awssdk.transfer.s3.DownloadRequest;
import software.amazon.awssdk.transfer.s3.S3ClientConfiguration;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.S3TransferManagerOverrideConfiguration;
import software.amazon.awssdk.transfer.s3.Upload;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer;
import software.amazon.awssdk.transfer.s3.UploadRequest;
import software.amazon.awssdk.utils.CompletableFutureUtils;
+import software.amazon.awssdk.utils.Validate;
@SdkInternalApi
public final class DefaultS3TransferManager implements S3TransferManager {
private final S3CrtAsyncClient s3CrtAsyncClient;
+ private final TransferManagerConfiguration transferConfiguration;
+ private final UploadDirectoryHelper uploadDirectoryManager;
public DefaultS3TransferManager(DefaultBuilder tmBuilder) {
- S3CrtAsyncClient.S3CrtAsyncClientBuilder clientBuilder = S3CrtAsyncClient.builder();
- if (tmBuilder.s3ClientConfiguration != null) {
- tmBuilder.s3ClientConfiguration.credentialsProvider().ifPresent(clientBuilder::credentialsProvider);
- tmBuilder.s3ClientConfiguration.maxConcurrency().ifPresent(clientBuilder::maxConcurrency);
- tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes);
- tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region);
- tmBuilder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps);
- tmBuilder.s3ClientConfiguration.asyncConfiguration().ifPresent(clientBuilder::asyncConfiguration);
- }
-
- s3CrtAsyncClient = clientBuilder.build();
+ transferConfiguration = resolveTransferManagerConfiguration(tmBuilder);
+ s3CrtAsyncClient = initializeS3CrtClient(tmBuilder);
+ uploadDirectoryManager = new UploadDirectoryHelper(transferConfiguration, this::upload);
}
@SdkTestInternalApi
- DefaultS3TransferManager(S3CrtAsyncClient s3CrtAsyncClient) {
+ DefaultS3TransferManager(S3CrtAsyncClient s3CrtAsyncClient,
+ UploadDirectoryHelper uploadDirectoryManager,
+ TransferManagerConfiguration configuration) {
this.s3CrtAsyncClient = s3CrtAsyncClient;
+ this.transferConfiguration = configuration;
+ this.uploadDirectoryManager = uploadDirectoryManager;
+ }
+
+ private TransferManagerConfiguration resolveTransferManagerConfiguration(DefaultBuilder tmBuilder) {
+ TransferManagerConfiguration.Builder transferConfigBuilder = TransferManagerConfiguration.builder();
+ tmBuilder.transferManagerConfiguration.uploadDirectoryConfiguration()
+ .ifPresent(transferConfigBuilder::uploadDirectoryConfiguration);
+ tmBuilder.transferManagerConfiguration.executor().ifPresent(transferConfigBuilder::executor);
+ return transferConfigBuilder.build();
+ }
+
+ private S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder) {
+ S3CrtAsyncClient.S3CrtAsyncClientBuilder clientBuilder = S3CrtAsyncClient.builder();
+ tmBuilder.s3ClientConfiguration.credentialsProvider().ifPresent(clientBuilder::credentialsProvider);
+ tmBuilder.s3ClientConfiguration.maxConcurrency().ifPresent(clientBuilder::maxConcurrency);
+ tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes);
+ tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region);
+ tmBuilder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps);
+ ClientAsyncConfiguration clientAsyncConfiguration =
+ ClientAsyncConfiguration.builder()
+ .advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR,
+ transferConfiguration.option(TransferConfigurationOption.EXECUTOR))
+ .build();
+ clientBuilder.asyncConfiguration(clientAsyncConfiguration);
+
+ return clientBuilder.build();
}
@Override
public Upload upload(UploadRequest uploadRequest) {
- PutObjectRequest putObjectRequest = uploadRequest.putObjectRequest();
- AsyncRequestBody requestBody = requestBodyFor(uploadRequest);
+ try {
+ Validate.paramNotNull(uploadRequest, "uploadRequest");
+ assertNotUnsupportedArn(uploadRequest.putObjectRequest().bucket(), "upload");
+
+ PutObjectRequest putObjectRequest = uploadRequest.putObjectRequest();
+ AsyncRequestBody requestBody = requestBodyFor(uploadRequest);
- CompletableFuture putObjFuture = s3CrtAsyncClient.putObject(putObjectRequest, requestBody);
+ CompletableFuture putObjFuture = s3CrtAsyncClient.putObject(putObjectRequest, requestBody);
- CompletableFuture future = putObjFuture.thenApply(r -> DefaultCompletedUpload.builder()
- .response(r)
- .build());
- return new DefaultUpload(CompletableFutureUtils.forwardExceptionTo(future, putObjFuture));
+ CompletableFuture future = putObjFuture.thenApply(r -> CompletedUpload.builder()
+ .response(r)
+ .build());
+ return new DefaultUpload(CompletableFutureUtils.forwardExceptionTo(future, putObjFuture));
+ } catch (Throwable throwable) {
+ return new DefaultUpload(CompletableFutureUtils.failedFuture(throwable));
+ }
}
@Override
- public Download download(DownloadRequest downloadRequest) {
- CompletableFuture getObjectFuture =
- s3CrtAsyncClient.getObject(downloadRequest.getObjectRequest(),
- AsyncResponseTransformer.toFile(downloadRequest.destination()));
- CompletableFuture future =
- getObjectFuture.thenApply(r -> DefaultCompletedDownload.builder().response(r).build());
+ public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) {
+ try {
+ Validate.paramNotNull(uploadDirectoryRequest, "uploadDirectoryRequest");
+ assertNotUnsupportedArn(uploadDirectoryRequest.bucket(), "uploadDirectory");
+
+ return uploadDirectoryManager.uploadDirectory(uploadDirectoryRequest);
+ } catch (Throwable throwable) {
+ return UploadDirectoryTransfer.builder().completionFuture(CompletableFutureUtils.failedFuture(throwable)).build();
+ }
+ }
- return new DefaultDownload(CompletableFutureUtils.forwardExceptionTo(future, getObjectFuture));
+ @Override
+ public Download download(DownloadRequest downloadRequest) {
+ try {
+ Validate.paramNotNull(downloadRequest, "downloadRequest");
+ assertNotUnsupportedArn(downloadRequest.getObjectRequest().bucket(), "download");
+
+ CompletableFuture getObjectFuture =
+ s3CrtAsyncClient.getObject(downloadRequest.getObjectRequest(),
+ AsyncResponseTransformer.toFile(downloadRequest.destination()));
+ CompletableFuture future =
+ getObjectFuture.thenApply(r -> CompletedDownload.builder().response(r).build());
+
+ return new DefaultDownload(CompletableFutureUtils.forwardExceptionTo(future, getObjectFuture));
+ } catch (Throwable throwable) {
+ return new DefaultDownload(CompletableFutureUtils.failedFuture(throwable));
+ }
}
@Override
public void close() {
s3CrtAsyncClient.close();
+ transferConfiguration.close();
}
public static Builder builder() {
return new DefaultBuilder();
}
+ private static void assertNotUnsupportedArn(String bucket, String operation) {
+ if (!bucket.startsWith("arn:")) {
+ return;
+ }
+
+ if (isObjectLambdaArn(bucket)) {
+ String error = String.format("%s does not support S3 Object Lambda resources", operation);
+ throw new IllegalArgumentException(error);
+ }
+
+ Arn arn = Arn.fromString(bucket);
+
+ if (isMrapArn(arn)) {
+ String error = String.format("%s does not support S3 multi-region access point ARN", operation);
+ throw new IllegalArgumentException(error);
+ }
+ }
+
+ private static boolean isObjectLambdaArn(String arn) {
+ return arn.contains(":s3-object-lambda");
+ }
+
+ private static boolean isMrapArn(Arn arn) {
+ S3Resource s3Resource = S3ArnConverter.create().convertArn(arn);
+
+ S3AccessPointResource s3EndpointResource =
+ Validate.isInstanceOf(S3AccessPointResource.class, s3Resource,
+ "An ARN was passed as a bucket parameter to an S3 operation, however it does not "
+ + "appear to be a valid S3 access point ARN.");
+
+ return !s3EndpointResource.region().isPresent();
+ }
+
private AsyncRequestBody requestBodyFor(UploadRequest uploadRequest) {
return AsyncRequestBody.fromFile(uploadRequest.source());
}
private static class DefaultBuilder implements S3TransferManager.Builder {
- private S3ClientConfiguration s3ClientConfiguration;
+ private S3ClientConfiguration s3ClientConfiguration = S3ClientConfiguration.builder().build();
+ private S3TransferManagerOverrideConfiguration transferManagerConfiguration =
+ S3TransferManagerOverrideConfiguration.builder().build();
+
+ private DefaultBuilder() {
+ }
@Override
public Builder s3ClientConfiguration(S3ClientConfiguration configuration) {
@@ -102,6 +200,12 @@ public Builder s3ClientConfiguration(S3ClientConfiguration configuration) {
return this;
}
+ @Override
+ public Builder transferConfiguration(S3TransferManagerOverrideConfiguration transferManagerConfiguration) {
+ this.transferManagerConfiguration = transferManagerConfiguration;
+ return this;
+ }
+
@Override
public S3TransferManager build() {
return new DefaultS3TransferManager(this);
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferConfigurationOption.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferConfigurationOption.java
new file mode 100644
index 000000000000..55a73dc41cc0
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferConfigurationOption.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import java.util.concurrent.Executor;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.utils.AttributeMap;
+
+/**
+ * A set of internal options required by the {@link DefaultS3TransferManager} via {@link TransferManagerConfiguration}.
+ * It contains the default settings
+ *
+ */
+@SdkInternalApi
+public final class TransferConfigurationOption extends AttributeMap.Key {
+ public static final TransferConfigurationOption UPLOAD_DIRECTORY_MAX_DEPTH =
+ new TransferConfigurationOption<>("UploadDirectoryMaxDepth", Integer.class);
+
+ public static final TransferConfigurationOption UPLOAD_DIRECTORY_RECURSIVE =
+ new TransferConfigurationOption<>("UploadDirectoryRecursive", Boolean.class);
+
+ public static final TransferConfigurationOption UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS =
+ new TransferConfigurationOption<>("UploadDirectoryFileVisitOption", Boolean.class);
+
+ public static final TransferConfigurationOption EXECUTOR =
+ new TransferConfigurationOption<>("Executor", Executor.class);
+
+ public static final String DEFAULT_DELIMITER = "/";
+
+ private static final int DEFAULT_UPLOAD_DIRECTORY_MAX_DEPTH = Integer.MAX_VALUE;
+ private static final Boolean DEFAULT_UPLOAD_DIRECTORY_RECURSIVE = Boolean.TRUE;
+
+ private static final Boolean DEFAULT_UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS = Boolean.FALSE;
+
+ // TODO: revisit default settings before GA
+ public static final AttributeMap TRANSFER_MANAGER_DEFAULTS = AttributeMap
+ .builder()
+ .put(UPLOAD_DIRECTORY_MAX_DEPTH, DEFAULT_UPLOAD_DIRECTORY_MAX_DEPTH)
+ .put(UPLOAD_DIRECTORY_RECURSIVE, DEFAULT_UPLOAD_DIRECTORY_RECURSIVE)
+ .put(UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS, DEFAULT_UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS)
+ .build();
+
+ private final String name;
+
+ private TransferConfigurationOption(String name, Class clzz) {
+ super(clzz);
+ this.name = name;
+ }
+
+ /**
+ * Note that the name is mainly used for debugging purposes. Two option key objects with the same name do not represent
+ * the same option. Option keys are compared by reference when obtaining a value from an {@link AttributeMap}.
+ *
+ * @return Name of this option key.
+ */
+ public String name() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+}
+
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfiguration.java
new file mode 100644
index 000000000000..05ca8a64d5e1
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfiguration.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.TRANSFER_MANAGER_DEFAULTS;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_MAX_DEPTH;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_RECURSIVE;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryOverrideConfiguration;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+import software.amazon.awssdk.utils.AttributeMap;
+import software.amazon.awssdk.utils.ExecutorUtils;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
+import software.amazon.awssdk.utils.ThreadFactoryBuilder;
+import software.amazon.awssdk.utils.Validate;
+
+/**
+ * Contains resolved configuration settings for {@link S3TransferManager}.
+ * This configuration object can be {@link #close()}d to release all closeable resources configured within it.
+ */
+@SdkInternalApi
+public class TransferManagerConfiguration implements SdkAutoCloseable {
+ private final AttributeMap options;
+
+ private TransferManagerConfiguration(Builder builder) {
+ UploadDirectoryOverrideConfiguration uploadDirectoryConfiguration =
+ Validate.paramNotNull(builder.uploadDirectoryOverrideConfiguration, "uploadDirectoryOverrideConfiguration");
+ AttributeMap.Builder standardOptions = AttributeMap.builder();
+
+ standardOptions.put(TransferConfigurationOption.UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS,
+ uploadDirectoryConfiguration.followSymbolicLinks().orElse(null));
+ standardOptions.put(TransferConfigurationOption.UPLOAD_DIRECTORY_MAX_DEPTH,
+ uploadDirectoryConfiguration.maxDepth().orElse(null));
+ standardOptions.put(TransferConfigurationOption.UPLOAD_DIRECTORY_RECURSIVE,
+ uploadDirectoryConfiguration.recursive().orElse(null));
+ finalizeExecutor(builder, standardOptions);
+
+ options = standardOptions.build().merge(TRANSFER_MANAGER_DEFAULTS);
+ }
+
+ private void finalizeExecutor(Builder builder, AttributeMap.Builder standardOptions) {
+ if (builder.executor != null) {
+ standardOptions.put(TransferConfigurationOption.EXECUTOR, ExecutorUtils.unmanagedExecutor(builder.executor));
+ } else {
+
+ standardOptions.put(TransferConfigurationOption.EXECUTOR, defaultExecutor());
+ }
+ }
+
+ /**
+ * Retrieve the value of a specific option.
+ */
+ public T option(TransferConfigurationOption option) {
+ return options.get(option);
+ }
+
+ public boolean resolveUploadDirectoryRecursive(UploadDirectoryRequest request) {
+ return request.overrideConfiguration()
+ .flatMap(UploadDirectoryOverrideConfiguration::recursive)
+ .orElse(options.get(UPLOAD_DIRECTORY_RECURSIVE));
+ }
+
+ public boolean resolveUploadDirectoryFollowSymbolicLinks(UploadDirectoryRequest request) {
+ return request.overrideConfiguration()
+ .flatMap(UploadDirectoryOverrideConfiguration::followSymbolicLinks)
+ .orElse(options.get(UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS));
+ }
+
+ public int resolveUploadDirectoryMaxDepth(UploadDirectoryRequest request) {
+ return request.overrideConfiguration()
+ .flatMap(UploadDirectoryOverrideConfiguration::maxDepth)
+ .orElse(options.get(UPLOAD_DIRECTORY_MAX_DEPTH));
+ }
+
+ @Override
+ public void close() {
+ options.close();
+ }
+
+ // TODO: revisit this before GA
+ private Executor defaultExecutor() {
+ int maxPoolSize = 100;
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(0, maxPoolSize,
+ 60, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(1_000),
+ new ThreadFactoryBuilder()
+ .threadNamePrefix("s3-transfer-manager").build());
+ // Allow idle core threads to time out
+ executor.allowCoreThreadTimeOut(true);
+ return executor;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static final class Builder {
+ private UploadDirectoryOverrideConfiguration uploadDirectoryOverrideConfiguration =
+ UploadDirectoryOverrideConfiguration.builder().build();
+ private Executor executor;
+
+ public Builder uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration configuration) {
+ this.uploadDirectoryOverrideConfiguration = configuration;
+ return this;
+ }
+
+ public Builder executor(Executor executor) {
+ this.executor = executor;
+ return this;
+ }
+
+ public TransferManagerConfiguration build() {
+ return new TransferManagerConfiguration(this);
+ }
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java
new file mode 100644
index 000000000000..64d64bddacc7
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelper.java
@@ -0,0 +1,238 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.DEFAULT_DELIMITER;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.annotations.SdkTestInternalApi;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.transfer.s3.CompletedUpload;
+import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory;
+import software.amazon.awssdk.transfer.s3.FailedFileUpload;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.Upload;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer;
+import software.amazon.awssdk.transfer.s3.UploadRequest;
+import software.amazon.awssdk.utils.CompletableFutureUtils;
+import software.amazon.awssdk.utils.Logger;
+import software.amazon.awssdk.utils.StringUtils;
+import software.amazon.awssdk.utils.Validate;
+
+/**
+ * An internal helper class that traverses the file tree and send the upload request
+ * for each file.
+ */
+@SdkInternalApi
+public class UploadDirectoryHelper {
+ private static final Logger log = Logger.loggerFor(S3TransferManager.class);
+
+ private final TransferManagerConfiguration transferConfiguration;
+ private final Function uploadFunction;
+ private final FileSystem fileSystem;
+
+ public UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration,
+ Function uploadFunction) {
+
+ this.transferConfiguration = transferConfiguration;
+ this.uploadFunction = uploadFunction;
+ this.fileSystem = FileSystems.getDefault();
+ }
+
+ @SdkTestInternalApi
+ UploadDirectoryHelper(TransferManagerConfiguration transferConfiguration,
+ Function uploadFunction,
+ FileSystem fileSystem) {
+
+ this.transferConfiguration = transferConfiguration;
+ this.uploadFunction = uploadFunction;
+ this.fileSystem = fileSystem;
+ }
+
+ public UploadDirectoryTransfer uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) {
+
+ CompletableFuture returnFuture = new CompletableFuture<>();
+
+ // offload the execution to the transfer manager executor
+ CompletableFuture.runAsync(() -> doUploadDirectory(returnFuture, uploadDirectoryRequest),
+ transferConfiguration.option(TransferConfigurationOption.EXECUTOR))
+ .whenComplete((r, t) -> {
+ if (t != null) {
+ returnFuture.completeExceptionally(t);
+ }
+ });
+
+ return UploadDirectoryTransfer.builder().completionFuture(returnFuture).build();
+ }
+
+ private void doUploadDirectory(CompletableFuture returnFuture,
+ UploadDirectoryRequest uploadDirectoryRequest) {
+
+ Path directory = uploadDirectoryRequest.sourceDirectory();
+
+ validateDirectory(uploadDirectoryRequest);
+
+ Collection failedUploads = new ConcurrentLinkedQueue<>();
+ List> futures;
+
+ try (Stream entries = listFiles(directory, uploadDirectoryRequest)) {
+ futures = entries.map(path -> {
+ CompletableFuture future = uploadSingleFile(uploadDirectoryRequest,
+ failedUploads, path);
+
+ // Forward cancellation of the return future to all individual futures.
+ CompletableFutureUtils.forwardExceptionTo(returnFuture, future);
+ return future;
+ }).collect(Collectors.toList());
+ }
+
+ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+ .whenComplete((r, t) -> returnFuture.complete(CompletedUploadDirectory.builder()
+ .failedUploads(failedUploads)
+ .build()));
+ }
+
+ private void validateDirectory(UploadDirectoryRequest uploadDirectoryRequest) {
+ Path directory = uploadDirectoryRequest.sourceDirectory();
+ Validate.isTrue(Files.exists(directory), "The source directory provided (%s) does not exist", directory);
+ boolean followSymbolicLinks = transferConfiguration.resolveUploadDirectoryFollowSymbolicLinks(uploadDirectoryRequest);
+ if (followSymbolicLinks) {
+ Validate.isTrue(Files.isDirectory(directory), "The source directory provided (%s) is not a "
+ + "directory", directory);
+ } else {
+ Validate.isTrue(Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS), "The source directory provided (%s)"
+ + " is not a "
+ + "directory", directory);
+ }
+ }
+
+ private CompletableFuture uploadSingleFile(UploadDirectoryRequest uploadDirectoryRequest,
+ Collection failedUploads,
+ Path path) {
+ int nameCount = uploadDirectoryRequest.sourceDirectory().getNameCount();
+ UploadRequest uploadRequest = constructUploadRequest(uploadDirectoryRequest, nameCount, path);
+ log.debug(() -> String.format("Sending upload request (%s) for path (%s)", uploadRequest, path));
+ CompletableFuture future = uploadFunction.apply(uploadRequest).completionFuture();
+ future.whenComplete((r, t) -> {
+ if (t != null) {
+ failedUploads.add(FailedFileUpload.builder()
+ .exception(t)
+ .request(uploadRequest)
+ .build());
+ }
+ });
+ return future;
+ }
+
+ private Stream listFiles(Path directory, UploadDirectoryRequest request) {
+
+ try {
+ boolean recursive = transferConfiguration.resolveUploadDirectoryRecursive(request);
+ boolean followSymbolicLinks = transferConfiguration.resolveUploadDirectoryFollowSymbolicLinks(request);
+
+ if (!recursive) {
+ return Files.list(directory)
+ .filter(p -> isRegularFile(p, followSymbolicLinks));
+ }
+
+ int maxDepth = transferConfiguration.resolveUploadDirectoryMaxDepth(request);
+
+ if (followSymbolicLinks) {
+ return Files.walk(directory, maxDepth, FileVisitOption.FOLLOW_LINKS)
+ .filter(path -> isRegularFile(path, true));
+ }
+
+ return Files.walk(directory, maxDepth)
+ .filter(path -> isRegularFile(path, false));
+
+ } catch (IOException e) {
+ throw SdkClientException.create("Failed to list files within the provided directory: " + directory, e);
+ }
+ }
+
+ private boolean isRegularFile(Path path, boolean followSymlinks) {
+ if (followSymlinks) {
+ return Files.isRegularFile(path);
+ }
+
+ return Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS);
+ }
+
+ /**
+ * If the prefix already ends with the same string as delimiter, there is no need to add delimiter.
+ */
+ private static String normalizePrefix(String prefix, String delimiter) {
+ if (StringUtils.isEmpty(prefix)) {
+ return "";
+ }
+ return prefix.endsWith(delimiter) ? prefix : prefix + delimiter;
+ }
+
+ private String getRelativePathName(int directoryNameCount, Path path, String delimiter) {
+ String relativePathName = path.subpath(directoryNameCount,
+ path.getNameCount()).toString();
+
+ String separator = fileSystem.getSeparator();
+
+ // Optimization for the case where separator equals to the delimiter: there is no need to call String#replace which
+ // invokes Pattern#compile in Java 8
+ if (delimiter.equals(separator)) {
+ return relativePathName;
+ }
+
+ return relativePathName.replace(separator, delimiter);
+ }
+
+ private UploadRequest constructUploadRequest(UploadDirectoryRequest uploadDirectoryRequest, int directoryNameCount,
+ Path path) {
+ String delimiter =
+ uploadDirectoryRequest.delimiter()
+ .filter(s -> !s.isEmpty())
+ .orElse(DEFAULT_DELIMITER);
+
+ String prefix = uploadDirectoryRequest.prefix()
+ .map(s -> normalizePrefix(s, delimiter))
+ .orElse("");
+
+ String relativePathName = getRelativePathName(directoryNameCount, path, delimiter);
+ String key = prefix + relativePathName;
+
+ PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+ .bucket(uploadDirectoryRequest.bucket())
+ .key(key)
+ .build();
+ return UploadRequest.builder()
+ .source(path)
+ .putObjectRequest(putObjectRequest)
+ .build();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java
new file mode 100644
index 000000000000..a815e1ea4081
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedDownloadTest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class CompletedDownloadTest {
+
+ @Test
+ public void responseNull_shouldThrowException() {
+ assertThatThrownBy(() -> CompletedDownload.builder().build()).isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("must not be null");
+ }
+
+ @Test
+ public void equalsHashcode() {
+ EqualsVerifier.forClass(CompletedDownload.class)
+ .withNonnullFields("response")
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java
new file mode 100644
index 000000000000..2e79f3fbe39f
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/CompletedUploadTest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class CompletedUploadTest {
+
+ @Test
+ public void responseNull_shouldThrowException() {
+ assertThatThrownBy(() -> CompletedUpload.builder().build()).isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("must not be null");
+ }
+
+ @Test
+ public void equalsHashcode() {
+ EqualsVerifier.forClass(CompletedUpload.class)
+ .withNonnullFields("response")
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java
index 36a293320d5f..6a54cd7e90e2 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java
@@ -20,10 +20,10 @@
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
+import nl.jqno.equalsverifier.EqualsVerifier;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import software.amazon.awssdk.services.s3.model.GetObjectRequest;
public class DownloadRequestTest {
@@ -75,30 +75,8 @@ public void usingFile_null_shouldThrowException() {
@Test
public void equals_hashcode() {
- GetObjectRequest getObjectRequest = GetObjectRequest.builder()
- .bucket("bucket")
- .key("key")
- .build();
-
- DownloadRequest request1 = DownloadRequest.builder()
- .getObjectRequest(b -> b.bucket("bucket").key("key"))
- .destination(Paths.get("."))
- .build();
-
- DownloadRequest request2 = DownloadRequest.builder()
- .getObjectRequest(getObjectRequest)
- .destination(Paths.get("."))
- .build();
-
- DownloadRequest request3 = DownloadRequest.builder()
- .getObjectRequest(b -> b.bucket("bucket1").key("key1"))
- .destination(Paths.get("."))
- .build();
-
- assertThat(request1).isEqualTo(request2);
- assertThat(request1.hashCode()).isEqualTo(request2.hashCode());
-
- assertThat(request1.hashCode()).isNotEqualTo(request3.hashCode());
- assertThat(request1).isNotEqualTo(request3);
+ EqualsVerifier.forClass(DownloadRequest.class)
+ .withNonnullFields("destination", "getObjectRequest")
+ .verify();
}
}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java
new file mode 100644
index 000000000000..a2bd8112b9e4
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/FailedSingleFileUploadTest.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.nio.file.Paths;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+import software.amazon.awssdk.core.exception.SdkClientException;
+
+public class FailedSingleFileUploadTest {
+
+ @Test
+ public void requestNull_mustThrowException() {
+ assertThatThrownBy(() -> FailedFileUpload.builder()
+ .exception(SdkClientException.create("xxx")).build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("request must not be null");
+ }
+
+ @Test
+ public void exceptionNull_mustThrowException() {
+ UploadRequest uploadRequest =
+ UploadRequest.builder().source(Paths.get(".")).putObjectRequest(p -> p.bucket("bucket").key("key")).build();
+ assertThatThrownBy(() -> FailedFileUpload.builder()
+ .request(uploadRequest).build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("exception must not be null");
+ }
+
+ @Test
+ public void equalsHashcode() {
+ EqualsVerifier.forClass(FailedFileUpload.class)
+ .withNonnullFields("exception", "request")
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java
index 9788da748654..ec2a2b67de6e 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java
@@ -19,11 +19,10 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;
+import nl.jqno.equalsverifier.EqualsVerifier;
import org.junit.Test;
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
import software.amazon.awssdk.regions.Region;
public class S3ClientConfigurationTest {
@@ -98,35 +97,7 @@ public void build_emptyBuilder() {
@Test
public void equalsHashCode() {
- AwsCredentialsProvider credentials = () -> AwsBasicCredentials.create("test"
- , "test");
- S3ClientConfiguration configuration1 = S3ClientConfiguration.builder()
- .credentialsProvider(credentials)
- .maxConcurrency(100)
- .targetThroughputInGbps(10.0)
- .region(Region.US_WEST_2)
- .minimumPartSizeInBytes(5 * MB)
- .build();
-
- S3ClientConfiguration configuration2 = S3ClientConfiguration.builder()
- .credentialsProvider(credentials)
- .maxConcurrency(100)
- .targetThroughputInGbps(10.0)
- .region(Region.US_WEST_2)
- .minimumPartSizeInBytes(5 * MB)
- .build();
-
- S3ClientConfiguration configuration3 = configuration1.toBuilder()
- .credentialsProvider(AnonymousCredentialsProvider.create())
- .maxConcurrency(50)
- .targetThroughputInGbps(1.0)
- .asyncConfiguration(c -> c.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR,
- Runnable::run))
- .build();
-
- assertThat(configuration1).isEqualTo(configuration2);
- assertThat(configuration1.hashCode()).isEqualTo(configuration2.hashCode());
- assertThat(configuration1).isNotEqualTo(configuration3);
- assertThat(configuration1.hashCode()).isNotEqualTo(configuration3.hashCode());
+ EqualsVerifier.forClass(S3ClientConfiguration.class)
+ .verify();
}
}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfigurationTest.java
new file mode 100644
index 000000000000..00fa560be571
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3TransferManagerOverrideConfigurationTest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.concurrent.Executor;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class S3TransferManagerOverrideConfigurationTest {
+
+ @Test
+ public void build_allProperties() {
+ Executor executor = Mockito.mock(Executor.class);
+ UploadDirectoryOverrideConfiguration directoryOverrideConfiguration =
+ UploadDirectoryOverrideConfiguration.builder()
+ .build();
+ S3TransferManagerOverrideConfiguration configuration =
+ S3TransferManagerOverrideConfiguration.builder()
+ .uploadDirectoryConfiguration(directoryOverrideConfiguration)
+ .executor(executor)
+ .build();
+
+ assertThat(configuration.executor()).contains(executor);
+ assertThat(configuration.uploadDirectoryConfiguration()).contains(directoryOverrideConfiguration);
+ }
+
+ @Test
+ public void build_emptyBuilder() {
+ S3TransferManagerOverrideConfiguration configuration = S3TransferManagerOverrideConfiguration.builder()
+ .build();
+
+ assertThat(configuration.executor()).isEmpty();
+ assertThat(configuration.uploadDirectoryConfiguration()).isEmpty();
+ }
+
+ @Test
+ public void equalsHashCode() {
+ EqualsVerifier.forClass(S3TransferManagerOverrideConfiguration.class)
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfigurationTest.java
new file mode 100644
index 000000000000..09a8fa590867
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryOverrideConfigurationTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class UploadDirectoryOverrideConfigurationTest {
+
+ @Test
+ public void maxDepthNonNegative_shouldThrowException() {
+ assertThatThrownBy(() -> UploadDirectoryOverrideConfiguration.builder()
+ .maxDepth(0)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("positive");
+
+ assertThatThrownBy(() -> UploadDirectoryOverrideConfiguration.builder()
+ .maxDepth(-1)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("positive");
+ }
+
+ @Test
+ public void defaultBuilder() {
+ UploadDirectoryOverrideConfiguration configuration = UploadDirectoryOverrideConfiguration.builder().build();
+ assertThat(configuration.followSymbolicLinks()).isEmpty();
+ assertThat(configuration.recursive()).isEmpty();
+ assertThat(configuration.maxDepth()).isEmpty();
+ }
+
+ @Test
+ public void defaultBuilderWithPropertySet() {
+ UploadDirectoryOverrideConfiguration configuration = UploadDirectoryOverrideConfiguration.builder()
+ .maxDepth(10)
+ .recursive(true)
+ .followSymbolicLinks(false)
+ .build();
+ assertThat(configuration.followSymbolicLinks()).contains(false);
+ assertThat(configuration.recursive()).contains(true);
+ assertThat(configuration.maxDepth()).contains(10);
+ }
+
+ @Test
+ public void equalsHashCode() {
+ EqualsVerifier.forClass(UploadDirectoryOverrideConfiguration.class).verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequestTest.java
new file mode 100644
index 000000000000..896bfc8d67f7
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadDirectoryRequestTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.nio.file.Paths;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+
+public class UploadDirectoryRequestTest {
+
+ @Test
+ public void noSourceDirectory_throws() {
+ assertThatThrownBy(() ->
+ UploadDirectoryRequest.builder().bucket("bucket").build()
+ ).isInstanceOf(NullPointerException.class).hasMessageContaining("sourceDirectory");
+ }
+
+ @Test
+ public void noBucket_throws() {
+ assertThatThrownBy(() ->
+ UploadDirectoryRequest.builder().sourceDirectory(Paths.get(".")).build()
+ ).isInstanceOf(NullPointerException.class).hasMessageContaining("bucket");
+ }
+
+ @Test
+ public void equals_hashcode() {
+ EqualsVerifier.forClass(UploadDirectoryRequest.class)
+ .withNonnullFields("sourceDirectory", "bucket")
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java
index 7ba0e7fc58f5..08debe7b8c1a 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java
@@ -20,6 +20,7 @@
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
+import nl.jqno.equalsverifier.EqualsVerifier;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -71,31 +72,9 @@ public void sourceUsingFile_null_shouldThrowException() {
@Test
public void equals_hashcode() {
- PutObjectRequest getObjectRequest = PutObjectRequest.builder()
- .bucket("bucket")
- .key("key")
- .build();
-
- UploadRequest request1 = UploadRequest.builder()
- .putObjectRequest(b -> b.bucket("bucket").key("key"))
- .source(Paths.get("."))
- .build();
-
- UploadRequest request2 = UploadRequest.builder()
- .putObjectRequest(getObjectRequest)
- .source(Paths.get("."))
- .build();
-
- UploadRequest request3 = UploadRequest.builder()
- .putObjectRequest(b -> b.bucket("bucket1").key("key1"))
- .source(Paths.get("."))
- .build();
-
- assertThat(request1).isEqualTo(request2);
- assertThat(request1.hashCode()).isEqualTo(request2.hashCode());
-
- assertThat(request1.hashCode()).isNotEqualTo(request3.hashCode());
- assertThat(request1).isNotEqualTo(request3);
+ EqualsVerifier.forClass(UploadRequest.class)
+ .withNonnullFields("source", "putObjectRequest")
+ .verify();
}
}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java
new file mode 100644
index 000000000000..64da9ab51145
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CompletedUploadDirectoryTest.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.junit.Test;
+import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory;
+
+public class CompletedUploadDirectoryTest {
+
+ @Test
+ public void equalsHashcode() {
+ EqualsVerifier.forClass(CompletedUploadDirectory.class)
+ .withNonnullFields("failedUploads")
+ .verify();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
index b8a8e423dc24..a1bad44562d4 100644
--- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java
@@ -16,36 +16,42 @@
package software.amazon.awssdk.transfer.s3.internal;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+
import java.nio.file.Paths;
import java.util.concurrent.CompletableFuture;
import org.junit.After;
import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.transfer.s3.CompletedDownload;
import software.amazon.awssdk.transfer.s3.CompletedUpload;
import software.amazon.awssdk.transfer.s3.DownloadRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
import software.amazon.awssdk.transfer.s3.UploadRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectRequest;
-import software.amazon.awssdk.services.s3.model.GetObjectResponse;
-import software.amazon.awssdk.services.s3.model.PutObjectRequest;
-import software.amazon.awssdk.services.s3.model.PutObjectResponse;
public class S3TransferManagerTest {
private S3CrtAsyncClient mockS3Crt;
private S3TransferManager tm;
+ private UploadDirectoryHelper uploadDirectoryManager;
+ private TransferManagerConfiguration configuration;
@Before
public void methodSetup() {
mockS3Crt = mock(S3CrtAsyncClient.class);
- tm = new DefaultS3TransferManager(mockS3Crt);
+ uploadDirectoryManager = mock(UploadDirectoryHelper.class);
+ configuration = mock(TransferManagerConfiguration.class);
+ tm = new DefaultS3TransferManager(mockS3Crt, uploadDirectoryManager, configuration);
}
@After
@@ -125,5 +131,75 @@ public void download_cancel_shouldForwardCancellation() {
assertThat(s3CrtFuture).isCancelled();
}
+ @Test
+ public void objectLambdaArnBucketProvided_shouldThrowException() {
+ String objectLambdaArn = "arn:xxx:s3-object-lambda";
+ assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(objectLambdaArn)
+ .key("key")).source(Paths.get(".")))
+ .completionFuture().join())
+ .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
+
+ assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(objectLambdaArn)
+ .key("key")).destination(Paths.get("."))).completionFuture().join())
+ .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
+
+ assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(objectLambdaArn).sourceDirectory(Paths.get("."))).completionFuture().join())
+ .hasMessageContaining("support S3 Object Lambda resources").hasCauseInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void mrapArnProvided_shouldThrowException() {
+ String mrapArn = "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap";
+ assertThatThrownBy(() -> tm.upload(b -> b.putObjectRequest(p -> p.bucket(mrapArn)
+ .key("key")).source(Paths.get(".")))
+ .completionFuture().join())
+ .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
+
+ assertThatThrownBy(() -> tm.download(b -> b.getObjectRequest(p -> p.bucket(mrapArn)
+ .key("key")).destination(Paths.get("."))).completionFuture().join())
+ .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
+
+ assertThatThrownBy(() -> tm.uploadDirectory(b -> b.bucket(mrapArn).sourceDirectory(Paths.get("."))).completionFuture().join())
+ .hasMessageContaining("multi-region access point ARN").hasCauseInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void uploadDirectory_throwException_shouldCompleteFutureExceptionally() {
+ RuntimeException exception = new RuntimeException("test");
+ when(uploadDirectoryManager.uploadDirectory(any(UploadDirectoryRequest.class))).thenThrow(exception);
+
+ assertThatThrownBy(() -> tm.uploadDirectory(u -> u.sourceDirectory(Paths.get("/"))
+ .bucket("bucketName")).completionFuture().join())
+ .hasCause(exception);
+ }
+
+ @Test
+ public void close_shouldCloseUnderlyingResources() {
+ S3TransferManager transferManager = new DefaultS3TransferManager(mockS3Crt, uploadDirectoryManager, configuration);
+ transferManager.close();
+ verify(mockS3Crt).close();
+ verify(configuration).close();
+ }
+ @Test
+ public void uploadDirectory_requestNull_shouldThrowException() {
+ UploadDirectoryRequest request = null;
+ assertThatThrownBy(() -> tm.uploadDirectory(request).completionFuture().join())
+ .hasCauseInstanceOf(NullPointerException.class)
+ .hasMessageContaining("must not be null");
+ }
+
+ @Test
+ public void upload_requestNull_shouldThrowException() {
+ UploadRequest request = null;
+ assertThatThrownBy(() -> tm.upload(request).completionFuture().join()).hasCauseInstanceOf(NullPointerException.class)
+ .hasMessageContaining("must not be null");
+ }
+
+ @Test
+ public void download_requestNull_shouldThrowException() {
+ DownloadRequest request = null;
+ assertThatThrownBy(() -> tm.download(request).completionFuture().join()).hasCauseInstanceOf(NullPointerException.class)
+ .hasMessageContaining("must not be null");
+ }
}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfigurationTest.java
new file mode 100644
index 000000000000..443dd4539cfa
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/TransferManagerConfigurationTest.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.EXECUTOR;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_MAX_DEPTH;
+import static software.amazon.awssdk.transfer.s3.internal.TransferConfigurationOption.UPLOAD_DIRECTORY_RECURSIVE;
+
+import java.nio.file.Paths;
+import java.util.concurrent.ExecutorService;
+import org.junit.Test;
+import org.mockito.Mockito;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryOverrideConfiguration;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+
+public class TransferManagerConfigurationTest {
+ private TransferManagerConfiguration transferManagerConfiguration;
+
+ @Test
+ public void resolveUploadDirectoryRecursive_requestOverride_requestOverrideShouldTakePrecedence() {
+ transferManagerConfiguration = TransferManagerConfiguration.builder()
+ .uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration.builder().recursive(true).build())
+ .build();
+ UploadDirectoryRequest uploadDirectoryRequest = UploadDirectoryRequest.builder()
+ .bucket("bucket")
+ .sourceDirectory(Paths.get("."))
+ .overrideConfiguration(o -> o.recursive(false))
+ .build();
+ assertThat(transferManagerConfiguration.resolveUploadDirectoryRecursive(uploadDirectoryRequest)).isFalse();
+ }
+
+ @Test
+ public void resolveMaxDepth_requestOverride_requestOverrideShouldTakePrecedence() {
+ transferManagerConfiguration = TransferManagerConfiguration.builder()
+ .uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration.builder()
+ .maxDepth(1)
+ .build())
+ .build();
+ UploadDirectoryRequest uploadDirectoryRequest = UploadDirectoryRequest.builder()
+ .bucket("bucket")
+ .sourceDirectory(Paths.get("."))
+ .overrideConfiguration(o -> o.maxDepth(2))
+ .build();
+ assertThat(transferManagerConfiguration.resolveUploadDirectoryMaxDepth(uploadDirectoryRequest)).isEqualTo(2);
+ }
+
+ @Test
+ public void resolveFollowSymlinks_requestOverride_requestOverrideShouldTakePrecedence() {
+ transferManagerConfiguration = TransferManagerConfiguration.builder()
+ .uploadDirectoryConfiguration(UploadDirectoryOverrideConfiguration.builder()
+ .followSymbolicLinks(false)
+ .build())
+ .build();
+ UploadDirectoryRequest uploadDirectoryRequest = UploadDirectoryRequest.builder()
+ .bucket("bucket")
+ .sourceDirectory(Paths.get("."))
+ .overrideConfiguration(o -> o.followSymbolicLinks(true))
+ .build();
+ assertThat(transferManagerConfiguration.resolveUploadDirectoryFollowSymbolicLinks(uploadDirectoryRequest)).isTrue();
+ }
+
+ @Test
+ public void noOverride_shouldUseDefaults() {
+ transferManagerConfiguration = TransferManagerConfiguration.builder().build();
+ assertThat(transferManagerConfiguration.option(UPLOAD_DIRECTORY_FOLLOW_SYMBOLIC_LINKS)).isFalse();
+ assertThat(transferManagerConfiguration.option(UPLOAD_DIRECTORY_MAX_DEPTH)).isEqualTo(Integer.MAX_VALUE);
+ assertThat(transferManagerConfiguration.option(UPLOAD_DIRECTORY_RECURSIVE)).isTrue();
+ assertThat(transferManagerConfiguration.option(EXECUTOR)).isNotNull();
+ }
+
+ @Test
+ public void close_noCustomExecutor_shouldCloseDefaultOne() {
+ transferManagerConfiguration = TransferManagerConfiguration.builder().build();
+ transferManagerConfiguration.close();
+ ExecutorService executor = (ExecutorService) transferManagerConfiguration.option(EXECUTOR);
+ assertThat(executor.isShutdown()).isTrue();
+ }
+
+ @Test
+ public void close_customExecutor_shouldNotCloseCustomExecutor() {
+ ExecutorService executorService = Mockito.mock(ExecutorService.class);
+ transferManagerConfiguration = TransferManagerConfiguration.builder().executor(executorService).build();
+ transferManagerConfiguration.close();
+ verify(executorService, never()).shutdown();
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java
new file mode 100644
index 000000000000..a41061c2c14c
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperParameterizedTest.java
@@ -0,0 +1,383 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.assertj.core.util.Sets;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.testutils.FileUtils;
+import software.amazon.awssdk.transfer.s3.CompletedUpload;
+import software.amazon.awssdk.transfer.s3.Upload;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer;
+import software.amazon.awssdk.transfer.s3.UploadRequest;
+import software.amazon.awssdk.utils.IoUtils;
+
+/**
+ * Testing {@link UploadDirectoryHelper} with different file systems.
+ */
+@RunWith(Parameterized.class)
+public class UploadDirectoryHelperParameterizedTest {
+ private static final Set FILE_SYSTEMS = Sets.newHashSet(Arrays.asList(Configuration.unix(),
+ Configuration.osX(),
+ Configuration.windows(),
+ Configuration.forCurrentPlatform()));
+ private Function singleUploadFunction;
+ private UploadDirectoryHelper uploadDirectoryHelper;
+ private Path directory;
+
+ @Parameterized.Parameter
+ public Configuration configuration;
+
+ private FileSystem jimfs;
+
+ @Parameterized.Parameters
+ public static Collection fileSystems() {
+ return FILE_SYSTEMS;
+ }
+
+ @Before
+ public void methodSetup() throws IOException {
+ singleUploadFunction = mock(Function.class);
+
+ if (!configuration.equals(Configuration.forCurrentPlatform())) {
+ jimfs = Jimfs.newFileSystem(configuration);
+ uploadDirectoryHelper = new UploadDirectoryHelper(TransferManagerConfiguration.builder().build(), singleUploadFunction, jimfs);
+ } else {
+ uploadDirectoryHelper = new UploadDirectoryHelper(TransferManagerConfiguration.builder().build(), singleUploadFunction);
+ }
+ directory = createTestDirectory();
+ }
+
+ @After
+ public void tearDown() {
+ if (jimfs != null) {
+ IoUtils.closeQuietly(jimfs, null);
+ } else {
+ FileUtils.cleanUpTestDirectory(directory);
+ }
+ }
+
+ @Test
+ public void uploadDirectory_defaultSetting_shouldRecursivelyUpload() {
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture()))
+ .thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .overrideConfiguration(o -> o.followSymbolicLinks(false))
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+ actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket"));
+
+ assertThat(actualRequests.size()).isEqualTo(3);
+
+ List keys =
+ actualRequests.stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys).containsOnly("bar.txt", "foo/1.txt", "foo/2.txt");
+ }
+
+ @Test
+ public void uploadDirectory_recursiveFalse_shouldOnlyUploadTopLevel() {
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .overrideConfiguration(o -> o.recursive(false))
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+
+ assertThat(actualRequests.size()).isEqualTo(1);
+
+ actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket"));
+ assertThat(actualRequests.get(0).putObjectRequest().key()).isEqualTo("bar.txt");
+ }
+
+ @Test
+ public void uploadDirectory_recursiveFalseFollowSymlinkTrue_shouldOnlyUploadTopLevel() {
+ // skip the test if we are using jimfs because it doesn't work well with symlink
+ assumeTrue(configuration.equals(Configuration.forCurrentPlatform()));
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .overrideConfiguration(o -> o.recursive(false).followSymbolicLinks(true))
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+ List keys =
+ actualRequests.stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys.size()).isEqualTo(2);
+ assertThat(keys).containsOnly("bar.txt", "symlink2");
+ }
+
+ @Test
+ public void uploadDirectory_FollowSymlinkTrue_shouldIncludeLinkedFiles() {
+ // skip the test if we are using jimfs because it doesn't work well with symlink
+ assumeTrue(configuration.equals(Configuration.forCurrentPlatform()));
+
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .overrideConfiguration(o -> o.followSymbolicLinks(true))
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+ actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket"));
+
+ List keys =
+ actualRequests.stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys.size()).isEqualTo(5);
+ assertThat(keys).containsOnly("bar.txt", "foo/1.txt", "foo/2.txt", "symlink/2.txt", "symlink2");
+ }
+
+ @Test
+ public void uploadDirectory_withPrefix_keysShouldHavePrefix() {
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .prefix("yolo")
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List keys =
+ requestArgumentCaptor.getAllValues().stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys.size()).isEqualTo(3);
+ keys.forEach(r -> assertThat(r).startsWith("yolo/"));
+ }
+
+ @Test
+ public void uploadDirectory_withDelimiter_shouldHonor() {
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .delimiter(",")
+ .prefix("yolo")
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List keys =
+ requestArgumentCaptor.getAllValues().stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys.size()).isEqualTo(3);
+ assertThat(keys).containsOnly("yolo,foo,2.txt", "yolo,foo,1.txt", "yolo,bar.txt");
+ }
+
+ @Test
+ public void uploadDirectory_maxLengthOne_shouldOnlyUploadTopLevel() {
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture()))
+ .thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .overrideConfiguration(o -> o.maxDepth(1))
+ .build());
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+ actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket"));
+
+ assertThat(actualRequests.size()).isEqualTo(1);
+
+ List keys =
+ actualRequests.stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys).containsOnly("bar.txt");
+ }
+
+
+ @Test
+ public void uploadDirectory_directoryNotExist_shouldCompleteFutureExceptionally() {
+ assertThatThrownBy(() -> uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder().sourceDirectory(Paths.get(
+ "randomstringneverexistas234ersaf1231"))
+ .bucket("bucketName").build()).completionFuture().join())
+ .hasMessageContaining("does not exist").hasCauseInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void uploadDirectory_notDirectory_shouldCompleteFutureExceptionally() {
+ // skip the test if we are using jimfs because it doesn't work well with symlink
+ assumeTrue(configuration.equals(Configuration.forCurrentPlatform()));
+ assertThatThrownBy(() -> uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder().sourceDirectory(Paths.get(directory.toString(), "symlink"))
+ .bucket("bucketName").build()).completionFuture().join())
+ .hasMessageContaining("is not a directory").hasCauseInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ public void uploadDirectory_notDirectoryFollowSymlinkTrue_shouldCompleteSuccessfully() {
+ // skip the test if we are using jimfs because it doesn't work well with symlink
+ assumeTrue(configuration.equals(Configuration.forCurrentPlatform()));
+ ArgumentCaptor requestArgumentCaptor = ArgumentCaptor.forClass(UploadRequest.class);
+
+ when(singleUploadFunction.apply(requestArgumentCaptor.capture())).thenReturn(completedUpload());
+ UploadDirectoryTransfer uploadDirectory = uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .overrideConfiguration(o -> o.followSymbolicLinks(true))
+ .sourceDirectory(Paths.get(directory.toString(), "symlink"))
+ .bucket("bucket").build());
+
+ uploadDirectory.completionFuture().join();
+
+ List actualRequests = requestArgumentCaptor.getAllValues();
+ actualRequests.forEach(r -> assertThat(r.putObjectRequest().bucket()).isEqualTo("bucket"));
+
+ assertThat(actualRequests.size()).isEqualTo(1);
+
+ List keys =
+ actualRequests.stream().map(u -> u.putObjectRequest().key())
+ .collect(Collectors.toList());
+
+ assertThat(keys).containsOnly("2.txt");
+ }
+
+ private DefaultUpload completedUpload() {
+ return new DefaultUpload(CompletableFuture.completedFuture(CompletedUpload.builder()
+ .response(PutObjectResponse.builder().build())
+ .build()));
+ }
+
+ private Path createTestDirectory() throws IOException {
+
+ if (jimfs != null) {
+ return createJmfsTestDirectory();
+ }
+
+ return createLocalTestDirectoryWithSymLink();
+ }
+
+ /**
+ * Create a test directory with the following structure
+ * - test1
+ * - foo
+ * - 1.txt
+ * - 2.txt
+ * - bar.txt
+ * - symlink -> test2
+ * - symlink2 -> test3/4.txt
+ * - test2
+ * - 2.txt
+ * - test3
+ * - 4.txt
+ */
+ private Path createLocalTestDirectoryWithSymLink() throws IOException {
+ Path directory = Files.createTempDirectory("test1");
+ Path anotherDirectory = Files.createTempDirectory("test2");
+ Path thirdDirectory = Files.createTempDirectory("test3");
+
+ String directoryName = directory.toString();
+ String anotherDirectoryName = anotherDirectory.toString();
+
+ Files.createDirectory(Paths.get(directory + "/foo"));
+
+ Files.write(Paths.get(directoryName, "bar.txt"), "bar".getBytes(StandardCharsets.UTF_8));
+ Files.write(Paths.get(directoryName, "foo/1.txt"), "1".getBytes(StandardCharsets.UTF_8));
+ Files.write(Paths.get(directoryName, "foo/2.txt"), "2".getBytes(StandardCharsets.UTF_8));
+
+ Files.write(Paths.get(anotherDirectoryName, "2.txt"), "2".getBytes(StandardCharsets.UTF_8));
+ Files.write(Paths.get(thirdDirectory.toString(), "3.txt"), "3".getBytes(StandardCharsets.UTF_8));
+
+ Files.createSymbolicLink(Paths.get(directoryName, "symlink"), anotherDirectory);
+ Files.createSymbolicLink(Paths.get(directoryName, "symlink2"), Paths.get(thirdDirectory.toString(), "3.txt"));
+ return directory;
+ }
+
+ /**
+ * Create a test directory with the following structure
+ * - test1
+ * - foo
+ * - 1.txt
+ * - 2.txt
+ * - bar.txt
+ */
+ private Path createJmfsTestDirectory() throws IOException {
+ String directoryName = "test";
+ Path directory = jimfs.getPath(directoryName);
+
+ Files.createDirectory(directory);
+
+ Files.createDirectory(jimfs.getPath(directoryName + "/foo"));
+
+ Files.write(jimfs.getPath(directoryName, "bar.txt"), "bar".getBytes(StandardCharsets.UTF_8));
+ Files.write(jimfs.getPath(directoryName, "foo/1.txt"), "1".getBytes(StandardCharsets.UTF_8));
+ Files.write(jimfs.getPath(directoryName, "foo/2.txt"), "2".getBytes(StandardCharsets.UTF_8));
+ return directory;
+ }
+}
diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java
new file mode 100644
index 000000000000..205bbb54dd0d
--- /dev/null
+++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/UploadDirectoryHelperTest.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.jimfs.Jimfs;
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.transfer.s3.CompletedUpload;
+import software.amazon.awssdk.transfer.s3.CompletedUploadDirectory;
+import software.amazon.awssdk.transfer.s3.Upload;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryRequest;
+import software.amazon.awssdk.transfer.s3.UploadDirectoryTransfer;
+import software.amazon.awssdk.transfer.s3.UploadRequest;
+
+public class UploadDirectoryHelperTest {
+ private static FileSystem jimfs;
+ private static Path directory;
+ private Function singleUploadFunction;
+ private UploadDirectoryHelper uploadDirectoryHelper;
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ jimfs = Jimfs.newFileSystem();
+ directory = jimfs.getPath("test");
+ Files.createDirectory(directory);
+ Files.createFile(jimfs.getPath("test/1"));
+ Files.createFile(jimfs.getPath("test/2"));
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ jimfs.close();
+ }
+
+ @Before
+ public void methodSetup() {
+ singleUploadFunction = mock(Function.class);
+ uploadDirectoryHelper = new UploadDirectoryHelper(TransferManagerConfiguration.builder().build(), singleUploadFunction);
+ }
+
+ @Test
+ public void uploadDirectory_cancel_shouldCancelAllFutures() {
+ CompletableFuture future = new CompletableFuture<>();
+ Upload upload = new DefaultUpload(future);
+
+ CompletableFuture future2 = new CompletableFuture<>();
+ Upload upload2 = new DefaultUpload(future2);
+
+ when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2);
+
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .build());
+
+ uploadDirectory.completionFuture().cancel(true);
+
+ assertThatThrownBy(() -> future.get(1, TimeUnit.SECONDS))
+ .isInstanceOf(CancellationException.class);
+
+ assertThatThrownBy(() -> future2.get(1, TimeUnit.SECONDS))
+ .isInstanceOf(CancellationException.class);
+ }
+
+ @Test
+ public void uploadDirectory_allUploadsSucceed_failedUploadsShouldBeEmpty() throws ExecutionException, InterruptedException,
+ TimeoutException {
+ PutObjectResponse putObjectResponse = PutObjectResponse.builder().eTag("1234").build();
+ CompletedUpload completedUpload = CompletedUpload.builder().response(putObjectResponse).build();
+ CompletableFuture successfulFuture = new CompletableFuture<>();
+
+ Upload upload = new DefaultUpload(successfulFuture);
+ successfulFuture.complete(completedUpload);
+
+ PutObjectResponse putObjectResponse2 = PutObjectResponse.builder().eTag("5678").build();
+ CompletedUpload completedUpload2 = CompletedUpload.builder().response(putObjectResponse2).build();
+ CompletableFuture failedFuture = new CompletableFuture<>();
+ Upload upload2 = new DefaultUpload(failedFuture);
+ failedFuture.complete(completedUpload2);
+
+ when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2);
+
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .build());
+
+ CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
+
+ assertThat(completedUploadDirectory.failedUploads()).isEmpty();
+ }
+
+ @Test
+ public void uploadDirectory_partialSuccess_shouldProvideFailedUploads() throws ExecutionException, InterruptedException,
+ TimeoutException {
+ PutObjectResponse putObjectResponse = PutObjectResponse.builder().eTag("1234").build();
+ CompletedUpload completedUpload = CompletedUpload.builder().response(putObjectResponse).build();
+ CompletableFuture successfulFuture = new CompletableFuture<>();
+ Upload upload = new DefaultUpload(successfulFuture);
+ successfulFuture.complete(completedUpload);
+
+ SdkClientException exception = SdkClientException.create("failed");
+ CompletableFuture failedFuture = new CompletableFuture<>();
+ Upload upload2 = new DefaultUpload(failedFuture);
+ failedFuture.completeExceptionally(exception);
+
+ when(singleUploadFunction.apply(any(UploadRequest.class))).thenReturn(upload, upload2);
+
+ UploadDirectoryTransfer uploadDirectory =
+ uploadDirectoryHelper.uploadDirectory(UploadDirectoryRequest.builder()
+ .sourceDirectory(directory)
+ .bucket("bucket")
+ .build());
+
+ CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
+
+ assertThat(completedUploadDirectory.failedUploads()).hasSize(1);
+ assertThat(completedUploadDirectory.failedUploads().iterator().next().exception()).isEqualTo(exception);
+ assertThat(completedUploadDirectory.failedUploads().iterator().next().request().source().toString()).isEqualTo("test/2");
+ }
+}
diff --git a/test/test-utils/src/main/java/software/amazon/awssdk/testutils/FileUtils.java b/test/test-utils/src/main/java/software/amazon/awssdk/testutils/FileUtils.java
new file mode 100644
index 000000000000..2b61a68e3d05
--- /dev/null
+++ b/test/test-utils/src/main/java/software/amazon/awssdk/testutils/FileUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.testutils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.stream.Stream;
+
+public final class FileUtils {
+ private FileUtils() {
+
+ }
+
+ public static void cleanUpTestDirectory(Path directory) {
+ try {
+ try (Stream paths = Files.walk(directory, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS)) {
+ paths.sorted(Comparator.reverseOrder())
+ .map(Path::toFile)
+ .forEach(File::delete);
+ }
+
+ } catch (IOException e) {
+ // ignore
+ e.printStackTrace();
+ }
+ }
+}