Support upload directory in s3 transfer manager

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java

@@ -17,6 +17,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
+import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -25,25 +26,30 @@
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.S3Object;
 import software.amazon.awssdk.testutils.FileUtils;
+import software.amazon.awssdk.utils.Logger;
 
 public class S3TransferManagerUploadDirectoryIntegrationTest extends S3IntegrationTestBase {
+    private static final Logger log = Logger.loggerFor(S3TransferManagerUploadDirectoryIntegrationTest.class);
     private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
 
     private static S3TransferManager tm;
     private static Path directory;
     private static S3Client s3Client;
+    private static String randomString;
 
     @BeforeClass
     public static void setUp() throws Exception {
         S3IntegrationTestBase.setUp();
         createBucket(TEST_BUCKET);
-
+        randomString = RandomStringUtils.random(100);
         directory = createLocalTestDirectory();
 
         tm = S3TransferManager.builder()
@@ -59,27 +65,63 @@ public static void setUp() throws Exception {
 
     @AfterClass
     public static void teardown() {
-        tm.close();
-        s3Client.close();
-        deleteBucketAndAllContents(TEST_BUCKET);
-        FileUtils.cleanUpTestDirectory(directory);
+        try {
+            FileUtils.cleanUpTestDirectory(directory);
+        } catch (Exception exception) {
+            log.warn(() -> "Failed to clean up test directory " + directory, exception);
+        }
+
+        try {
+            deleteBucketAndAllContents(TEST_BUCKET);
+        } catch (Exception exception) {
+            log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET, exception);
+        }
+
+        closeQuietly(tm, log.logger());
+        closeQuietly(s3Client, log.logger());
         S3IntegrationTestBase.cleanUp();
     }
 
     @Test
     public void uploadDirectory_filesSentCorrectly() {
         String prefix = "yolo";
-        UploadDirectory uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
-                                                                   .bucket(TEST_BUCKET)
-                                                                   .prefix(prefix)
-                                                                   .overrideConfiguration(o -> o.recursive(true)));
-        uploadDirectory.completionFuture().join();
+        UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
+                                                                           .bucket(TEST_BUCKET)
+                                                                           .prefix(prefix)
+                                                                           .overrideConfiguration(o -> o.recursive(true)));
+        CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
+        assertThat(completedUploadDirectory.failedUploads()).isEmpty();
 
         List<String> keys =
             s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
                     .collect(Collectors.toList());
 
         assertThat(keys).containsOnly(prefix + "/bar.txt", prefix + "/foo/1.txt", prefix + "/foo/2.txt");
+
+        keys.forEach(k -> verifyContent(k, k.substring(prefix.length() + 1) + randomString));
+    }
+
+    @Test
+    public void uploadDirectory_withDelimiter_filesSentCorrectly() {
+        String prefix = "hello";
+        String delimiter = "0";
+        UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
+                                                                           .bucket(TEST_BUCKET)
+                                                                           .delimiter(delimiter)
+                                                                           .prefix(prefix)
+                                                                           .overrideConfiguration(o -> o.recursive(true)));
+        CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
+        assertThat(completedUploadDirectory.failedUploads()).isEmpty();
+
+        List<String> keys =
+            s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
+                    .collect(Collectors.toList());
+
+        assertThat(keys).containsOnly(prefix + "0bar.txt", prefix + "0foo01.txt", prefix + "0foo02.txt");
+        keys.forEach(k -> {
+            String path = k.replace(delimiter, "/");
+            verifyContent(k, path.substring(prefix.length() + 1) + randomString);
+        });
     }
 
     private static Path createLocalTestDirectory() throws IOException {
@@ -88,11 +130,17 @@ private static Path createLocalTestDirectory() throws IOException {
         String directoryName = directory.toString();
 
         Files.createDirectory(Paths.get(directory + "/foo"));
-
-        Files.write(Paths.get(directoryName, "bar.txt"), "bar".getBytes(StandardCharsets.UTF_8));
-        Files.write(Paths.get(directoryName, "foo/1.txt"), "1".getBytes(StandardCharsets.UTF_8));
-        Files.write(Paths.get(directoryName, "foo/2.txt"), "2".getBytes(StandardCharsets.UTF_8));
+        Files.write(Paths.get(directoryName, "bar.txt"), ("bar.txt" + randomString).getBytes(StandardCharsets.UTF_8));
+        Files.write(Paths.get(directoryName, "foo/1.txt"), ("foo/1.txt" + randomString).getBytes(StandardCharsets.UTF_8));
+        Files.write(Paths.get(directoryName, "foo/2.txt"), ("foo/2.txt" + randomString).getBytes(StandardCharsets.UTF_8));
 
         return directory;
     }
+
+    private static void verifyContent(String key, String expectedContent) {
+        String actualContent = s3.getObject(r -> r.bucket(TEST_BUCKET).key(key),
+                                      ResponseTransformer.toBytes()).asUtf8String();
+
+        assertThat(actualContent).isEqualTo(expectedContent);
+    }
 }

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedUpload.java → services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileTransfer.java

@@ -20,19 +20,15 @@
 import software.amazon.awssdk.annotations.SdkPublicApi;
 
 /**
- *  A failed single file upload transfer.
+ * Represents a completed file transfer.
  */
 @SdkPublicApi
 @SdkPreviewApi
-public interface FailedUpload {
+public interface CompletedFileTransfer extends CompletedTransfer {
 
     /**
-     * @return the exception thrown
-     */
-    Throwable exception();
-
-    /**
-     * @return the path to the file that the transfer manager failed to upload
+     * Returns the path of the file that was upload from or downloaded to
+     * @return the path
      */
     Path path();
 }

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java

@@ -15,26 +15,109 @@
 
 package software.amazon.awssdk.transfer.s3;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import software.amazon.awssdk.annotations.SdkPreviewApi;
 import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.ToString;
+import software.amazon.awssdk.utils.Validate;
 
 /**
- * A completed download directory transfer.
+ * Represents a completed upload directory transfer to Amazon S3. It can be used to track
+ * failed single file uploads.
+ *
+ * @see S3TransferManager#uploadDirectory(UploadDirectoryRequest)
  */
 @SdkPublicApi
 @SdkPreviewApi
-public interface CompletedUploadDirectory extends CompletedTransfer {
+public final class CompletedUploadDirectory implements CompletedTransfer {
+    private final List<FailedSingleFileUpload> failedUploads;
+
+    private CompletedUploadDirectory(DefaultBuilder builder) {
+        this.failedUploads = Collections.unmodifiableList(new ArrayList<>(Validate.paramNotNull(builder.failedUploads,
+                                                                                                "failedUploads")));
+    }
 
     /**
-     * A list of failed single file uploads associated with one upload directory transfer
-     * @return A list of failed uploads
+     * Return failed uploads with error details, request metadata about each file that is failed to upload.
+     *
+     * @return a list of failed uploads
      */
-    List<FailedUpload> failedUploads();
+    public Collection<FailedSingleFileUpload> failedUploads() {
+        return failedUploads;
+    }
 
     /**
-     * A list of successful single file uploads associated with one upload directory transfer
-     * @return a list of successful  uploads
+     * Creates a default builder for {@link CompletedUpload}.
      */
-    List<CompletedUpload> successfulObjects();
+    public static Builder builder() {
+        return new DefaultBuilder();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        CompletedUploadDirectory that = (CompletedUploadDirectory) o;
+
+        return failedUploads.equals(that.failedUploads);
+    }
+
+    @Override
+    public int hashCode() {
+        return failedUploads.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return ToString.builder("CompletedUploadDirectory")
+                       .add("failedUploads", failedUploads)
+                       .build();
+    }
+
+    public interface Builder {
+
+        /**
+         * Sets a collection of {@link FailedSingleFileUpload}s
+         *
+         * @param failedUploads failed uploads
+         * @return This builder for method chaining.
+         */
+        Builder failedUploads(Collection<FailedSingleFileUpload> failedUploads);
+
+        /**
+         * Builds a {@link CompletedUploadDirectory} based on the properties supplied to this builder
+         * @return An initialized {@link CompletedUploadDirectory}
+         */
+        CompletedUploadDirectory build();
+    }
+
+    private static final class DefaultBuilder implements Builder {
+        private Collection<FailedSingleFileUpload> failedUploads = Collections.emptyList();
+
+        private DefaultBuilder() {
+        }
+
+        @Override
+        public Builder failedUploads(Collection<FailedSingleFileUpload> failedUploads) {
+            this.failedUploads = failedUploads;
+            return this;
+        }
+
+        public void setFailedUploads(Collection<FailedSingleFileUpload> failedUploads) {
+            failedUploads(failedUploads);
+        }
+
+        @Override
+        public CompletedUploadDirectory build() {
+            return new CompletedUploadDirectory(this);
+        }
+    }
 }

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedSingleFileTransfer.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+
+/**
+ * Represents a failed single file transfer in a multi-file transfer operation such as
+ * {@link S3TransferManager#uploadDirectory}
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public interface FailedSingleFileTransfer<T extends TransferRequest> {
+
+    /**
+     * The exception thrown from a specific single file transfer
+     *
+     * @return the exception thrown
+     */
+    Throwable exception();
+
+    /**
+     * The failed {@link TransferRequest}.
+     *
+     * @return the failed request
+     */
+    T request();
+
+    interface Builder<T extends TransferRequest> {
+        /**
+         * Specify the exception thrown from a specific single file transfer
+         *
+         * @param exception the exception thrown
+         * @return this builder for method chaining.
+         */
+        Builder<T> exception(Throwable exception);
+
+        /**
+         * Specify the failed request
+         *
+         * @param request the failed request
+         * @return this builder for method chaining.
+         */
+        Builder<T> request(T request);
+
+        /**
+         * Builds a {@link FailedSingleFileTransfer} based on the properties supplied to this builder
+         *
+         * @return An initialized {@link FailedSingleFileTransfer}
+         */
+        FailedSingleFileTransfer<T> build();
+    }
+}