Skip to content

Commit 144b16d

Browse files
zoewanggzoewangg
committed
address part of the feedback
1 parent c47d69e commit 144b16d

30 files changed

+1374
-714
lines changed

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java

+62-14
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import static org.assertj.core.api.Assertions.assertThat;
1919
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
2021

2122
import java.io.IOException;
2223
import java.nio.charset.StandardCharsets;
@@ -25,25 +26,30 @@
2526
import java.nio.file.Paths;
2627
import java.util.List;
2728
import java.util.stream.Collectors;
29+
import org.apache.commons.lang3.RandomStringUtils;
2830
import org.junit.AfterClass;
2931
import org.junit.BeforeClass;
3032
import org.junit.Test;
33+
import software.amazon.awssdk.core.sync.ResponseTransformer;
3134
import software.amazon.awssdk.services.s3.S3Client;
3235
import software.amazon.awssdk.services.s3.model.S3Object;
3336
import software.amazon.awssdk.testutils.FileUtils;
37+
import software.amazon.awssdk.utils.Logger;
3438

3539
public class S3TransferManagerUploadDirectoryIntegrationTest extends S3IntegrationTestBase {
40+
private static final Logger log = Logger.loggerFor(S3TransferManagerUploadDirectoryIntegrationTest.class);
3641
private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
3742

3843
private static S3TransferManager tm;
3944
private static Path directory;
4045
private static S3Client s3Client;
46+
private static String randomString;
4147

4248
@BeforeClass
4349
public static void setUp() throws Exception {
4450
S3IntegrationTestBase.setUp();
4551
createBucket(TEST_BUCKET);
46-
52+
randomString = RandomStringUtils.random(100);
4753
directory = createLocalTestDirectory();
4854

4955
tm = S3TransferManager.builder()
@@ -59,27 +65,63 @@ public static void setUp() throws Exception {
5965

6066
@AfterClass
6167
public static void teardown() {
62-
tm.close();
63-
s3Client.close();
64-
deleteBucketAndAllContents(TEST_BUCKET);
65-
FileUtils.cleanUpTestDirectory(directory);
68+
try {
69+
FileUtils.cleanUpTestDirectory(directory);
70+
} catch (Exception exception) {
71+
log.warn(() -> "Failed to clean up test directory " + directory, exception);
72+
}
73+
74+
try {
75+
deleteBucketAndAllContents(TEST_BUCKET);
76+
} catch (Exception exception) {
77+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET, exception);
78+
}
79+
80+
closeQuietly(tm, log.logger());
81+
closeQuietly(s3Client, log.logger());
6682
S3IntegrationTestBase.cleanUp();
6783
}
6884

6985
@Test
7086
public void uploadDirectory_filesSentCorrectly() {
7187
String prefix = "yolo";
72-
UploadDirectory uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
73-
.bucket(TEST_BUCKET)
74-
.prefix(prefix)
75-
.overrideConfiguration(o -> o.recursive(true)));
76-
uploadDirectory.completionFuture().join();
88+
UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
89+
.bucket(TEST_BUCKET)
90+
.prefix(prefix)
91+
.overrideConfiguration(o -> o.recursive(true)));
92+
CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
93+
assertThat(completedUploadDirectory.failedUploads()).isEmpty();
7794

7895
List<String> keys =
7996
s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
8097
.collect(Collectors.toList());
8198

8299
assertThat(keys).containsOnly(prefix + "/bar.txt", prefix + "/foo/1.txt", prefix + "/foo/2.txt");
100+
101+
keys.forEach(k -> verifyContent(k, k.substring(prefix.length() + 1) + randomString));
102+
}
103+
104+
@Test
105+
public void uploadDirectory_withDelimiter_filesSentCorrectly() {
106+
String prefix = "hello";
107+
String delimiter = "0";
108+
UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
109+
.bucket(TEST_BUCKET)
110+
.delimiter(delimiter)
111+
.prefix(prefix)
112+
.overrideConfiguration(o -> o.recursive(true)));
113+
CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
114+
assertThat(completedUploadDirectory.failedUploads()).isEmpty();
115+
116+
List<String> keys =
117+
s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
118+
.collect(Collectors.toList());
119+
120+
assertThat(keys).containsOnly(prefix + "0bar.txt", prefix + "0foo01.txt", prefix + "0foo02.txt");
121+
keys.forEach(k -> {
122+
String path = k.replace(delimiter, "/");
123+
verifyContent(k, path.substring(prefix.length() + 1) + randomString);
124+
});
83125
}
84126

85127
private static Path createLocalTestDirectory() throws IOException {
@@ -88,11 +130,17 @@ private static Path createLocalTestDirectory() throws IOException {
88130
String directoryName = directory.toString();
89131

90132
Files.createDirectory(Paths.get(directory + "/foo"));
91-
92-
Files.write(Paths.get(directoryName, "bar.txt"), "bar".getBytes(StandardCharsets.UTF_8));
93-
Files.write(Paths.get(directoryName, "foo/1.txt"), "1".getBytes(StandardCharsets.UTF_8));
94-
Files.write(Paths.get(directoryName, "foo/2.txt"), "2".getBytes(StandardCharsets.UTF_8));
133+
Files.write(Paths.get(directoryName, "bar.txt"), ("bar.txt" + randomString).getBytes(StandardCharsets.UTF_8));
134+
Files.write(Paths.get(directoryName, "foo/1.txt"), ("foo/1.txt" + randomString).getBytes(StandardCharsets.UTF_8));
135+
Files.write(Paths.get(directoryName, "foo/2.txt"), ("foo/2.txt" + randomString).getBytes(StandardCharsets.UTF_8));
95136

96137
return directory;
97138
}
139+
140+
private static void verifyContent(String key, String expectedContent) {
141+
String actualContent = s3.getObject(r -> r.bucket(TEST_BUCKET).key(key),
142+
ResponseTransformer.toBytes()).asUtf8String();
143+
144+
assertThat(actualContent).isEqualTo(expectedContent);
145+
}
98146
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedUpload.java renamed to ‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedFileTransfer.java

+4-8
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,15 @@
2020
import software.amazon.awssdk.annotations.SdkPublicApi;
2121

2222
/**
23-
* A failed single file upload transfer.
23+
* Represents a completed file transfer.
2424
*/
2525
@SdkPublicApi
2626
@SdkPreviewApi
27-
public interface FailedUpload {
27+
public interface CompletedFileTransfer extends CompletedTransfer {
2828

2929
/**
30-
* @return the exception thrown
31-
*/
32-
Throwable exception();
33-
34-
/**
35-
* @return the path to the file that the transfer manager failed to upload
30+
* Returns the path of the file that was upload from or downloaded to
31+
* @return the path
3632
*/
3733
Path path();
3834
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java

+91-8
Original file line numberDiff line numberDiff line change
@@ -15,26 +15,109 @@
1515

1616
package software.amazon.awssdk.transfer.s3;
1717

18+
import java.util.ArrayList;
19+
import java.util.Collection;
20+
import java.util.Collections;
1821
import java.util.List;
1922
import software.amazon.awssdk.annotations.SdkPreviewApi;
2023
import software.amazon.awssdk.annotations.SdkPublicApi;
24+
import software.amazon.awssdk.utils.ToString;
25+
import software.amazon.awssdk.utils.Validate;
2126

2227
/**
23-
* A completed download directory transfer.
28+
* Represents a completed upload directory transfer to Amazon S3. It can be used to track
29+
* failed single file uploads.
30+
*
31+
* @see S3TransferManager#uploadDirectory(UploadDirectoryRequest)
2432
*/
2533
@SdkPublicApi
2634
@SdkPreviewApi
27-
public interface CompletedUploadDirectory extends CompletedTransfer {
35+
public final class CompletedUploadDirectory implements CompletedTransfer {
36+
private final List<FailedSingleFileUpload> failedUploads;
37+
38+
private CompletedUploadDirectory(DefaultBuilder builder) {
39+
this.failedUploads = Collections.unmodifiableList(new ArrayList<>(Validate.paramNotNull(builder.failedUploads,
40+
"failedUploads")));
41+
}
2842

2943
/**
30-
* A list of failed single file uploads associated with one upload directory transfer
31-
* @return A list of failed uploads
44+
* Return failed uploads with error details, request metadata about each file that is failed to upload.
45+
*
46+
* @return a list of failed uploads
3247
*/
33-
List<FailedUpload> failedUploads();
48+
public Collection<FailedSingleFileUpload> failedUploads() {
49+
return failedUploads;
50+
}
3451

3552
/**
36-
* A list of successful single file uploads associated with one upload directory transfer
37-
* @return a list of successful uploads
53+
* Creates a default builder for {@link CompletedUpload}.
3854
*/
39-
List<CompletedUpload> successfulObjects();
55+
public static Builder builder() {
56+
return new DefaultBuilder();
57+
}
58+
59+
@Override
60+
public boolean equals(Object o) {
61+
if (this == o) {
62+
return true;
63+
}
64+
if (o == null || getClass() != o.getClass()) {
65+
return false;
66+
}
67+
68+
CompletedUploadDirectory that = (CompletedUploadDirectory) o;
69+
70+
return failedUploads.equals(that.failedUploads);
71+
}
72+
73+
@Override
74+
public int hashCode() {
75+
return failedUploads.hashCode();
76+
}
77+
78+
@Override
79+
public String toString() {
80+
return ToString.builder("CompletedUploadDirectory")
81+
.add("failedUploads", failedUploads)
82+
.build();
83+
}
84+
85+
public interface Builder {
86+
87+
/**
88+
* Sets a collection of {@link FailedSingleFileUpload}s
89+
*
90+
* @param failedUploads failed uploads
91+
* @return This builder for method chaining.
92+
*/
93+
Builder failedUploads(Collection<FailedSingleFileUpload> failedUploads);
94+
95+
/**
96+
* Builds a {@link CompletedUploadDirectory} based on the properties supplied to this builder
97+
* @return An initialized {@link CompletedUploadDirectory}
98+
*/
99+
CompletedUploadDirectory build();
100+
}
101+
102+
private static final class DefaultBuilder implements Builder {
103+
private Collection<FailedSingleFileUpload> failedUploads = Collections.emptyList();
104+
105+
private DefaultBuilder() {
106+
}
107+
108+
@Override
109+
public Builder failedUploads(Collection<FailedSingleFileUpload> failedUploads) {
110+
this.failedUploads = failedUploads;
111+
return this;
112+
}
113+
114+
public void setFailedUploads(Collection<FailedSingleFileUpload> failedUploads) {
115+
failedUploads(failedUploads);
116+
}
117+
118+
@Override
119+
public CompletedUploadDirectory build() {
120+
return new CompletedUploadDirectory(this);
121+
}
122+
}
40123
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedSingleFileTransfer.java

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import software.amazon.awssdk.annotations.SdkPreviewApi;
19+
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
21+
/**
22+
* Represents a failed single file transfer in a multi-file transfer operation such as
23+
* {@link S3TransferManager#uploadDirectory}
24+
*/
25+
@SdkPublicApi
26+
@SdkPreviewApi
27+
public interface FailedSingleFileTransfer<T extends TransferRequest> {
28+
29+
/**
30+
* The exception thrown from a specific single file transfer
31+
*
32+
* @return the exception thrown
33+
*/
34+
Throwable exception();
35+
36+
/**
37+
* The failed {@link TransferRequest}.
38+
*
39+
* @return the failed request
40+
*/
41+
T request();
42+
43+
interface Builder<T extends TransferRequest> {
44+
/**
45+
* Specify the exception thrown from a specific single file transfer
46+
*
47+
* @param exception the exception thrown
48+
* @return this builder for method chaining.
49+
*/
50+
Builder<T> exception(Throwable exception);
51+
52+
/**
53+
* Specify the failed request
54+
*
55+
* @param request the failed request
56+
* @return this builder for method chaining.
57+
*/
58+
Builder<T> request(T request);
59+
60+
/**
61+
* Builds a {@link FailedSingleFileTransfer} based on the properties supplied to this builder
62+
*
63+
* @return An initialized {@link FailedSingleFileTransfer}
64+
*/
65+
FailedSingleFileTransfer<T> build();
66+
}
67+
}

0 commit comments

Comments
 (0)