Skip to content

Commit c47d69e

Browse files
zoewanggzoewangg
committed
Support upload directory in s3 transfer manager
1 parent 49ce75a commit c47d69e

26 files changed

+2122
-19
lines changed

‎services-custom/s3-transfer-manager/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,11 @@
153153
<artifactId>reactive-streams-tck</artifactId>
154154
<scope>test</scope>
155155
</dependency>
156+
<dependency>
157+
<groupId>com.google.jimfs</groupId>
158+
<artifactId>jimfs</artifactId>
159+
<scope>test</scope>
160+
</dependency>
156161
</dependencies>
157162

158163
<build>

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
21+
import java.io.IOException;
22+
import java.nio.charset.StandardCharsets;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.nio.file.Paths;
26+
import java.util.List;
27+
import java.util.stream.Collectors;
28+
import org.junit.AfterClass;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
import software.amazon.awssdk.services.s3.S3Client;
32+
import software.amazon.awssdk.services.s3.model.S3Object;
33+
import software.amazon.awssdk.testutils.FileUtils;
34+
35+
public class S3TransferManagerUploadDirectoryIntegrationTest extends S3IntegrationTestBase {
36+
private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
37+
38+
private static S3TransferManager tm;
39+
private static Path directory;
40+
private static S3Client s3Client;
41+
42+
@BeforeClass
43+
public static void setUp() throws Exception {
44+
S3IntegrationTestBase.setUp();
45+
createBucket(TEST_BUCKET);
46+
47+
directory = createLocalTestDirectory();
48+
49+
tm = S3TransferManager.builder()
50+
.s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
51+
.region(DEFAULT_REGION)
52+
.maxConcurrency(100))
53+
.build();
54+
55+
s3Client = S3Client.builder()
56+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN).region(DEFAULT_REGION)
57+
.build();
58+
}
59+
60+
@AfterClass
61+
public static void teardown() {
62+
tm.close();
63+
s3Client.close();
64+
deleteBucketAndAllContents(TEST_BUCKET);
65+
FileUtils.cleanUpTestDirectory(directory);
66+
S3IntegrationTestBase.cleanUp();
67+
}
68+
69+
@Test
70+
public void uploadDirectory_filesSentCorrectly() {
71+
String prefix = "yolo";
72+
UploadDirectory uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
73+
.bucket(TEST_BUCKET)
74+
.prefix(prefix)
75+
.overrideConfiguration(o -> o.recursive(true)));
76+
uploadDirectory.completionFuture().join();
77+
78+
List<String> keys =
79+
s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
80+
.collect(Collectors.toList());
81+
82+
assertThat(keys).containsOnly(prefix + "/bar.txt", prefix + "/foo/1.txt", prefix + "/foo/2.txt");
83+
}
84+
85+
private static Path createLocalTestDirectory() throws IOException {
86+
Path directory = Files.createTempDirectory("test");
87+
88+
String directoryName = directory.toString();
89+
90+
Files.createDirectory(Paths.get(directory + "/foo"));
91+
92+
Files.write(Paths.get(directoryName, "bar.txt"), "bar".getBytes(StandardCharsets.UTF_8));
93+
Files.write(Paths.get(directoryName, "foo/1.txt"), "1".getBytes(StandardCharsets.UTF_8));
94+
Files.write(Paths.get(directoryName, "foo/2.txt"), "2".getBytes(StandardCharsets.UTF_8));
95+
96+
return directory;
97+
}
98+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUploadDirectory.java

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import java.util.List;
19+
import software.amazon.awssdk.annotations.SdkPreviewApi;
20+
import software.amazon.awssdk.annotations.SdkPublicApi;
21+
22+
/**
23+
* A completed download directory transfer.
24+
*/
25+
@SdkPublicApi
26+
@SdkPreviewApi
27+
public interface CompletedUploadDirectory extends CompletedTransfer {
28+
29+
/**
30+
* A list of failed single file uploads associated with one upload directory transfer
31+
* @return A list of failed uploads
32+
*/
33+
List<FailedUpload> failedUploads();
34+
35+
/**
36+
* A list of successful single file uploads associated with one upload directory transfer
37+
* @return a list of successful uploads
38+
*/
39+
List<CompletedUpload> successfulObjects();
40+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/FailedUpload.java

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import java.nio.file.Path;
19+
import software.amazon.awssdk.annotations.SdkPreviewApi;
20+
import software.amazon.awssdk.annotations.SdkPublicApi;
21+
22+
/**
23+
* A failed single file upload transfer.
24+
*/
25+
@SdkPublicApi
26+
@SdkPreviewApi
27+
public interface FailedUpload {
28+
29+
/**
30+
* @return the exception thrown
31+
*/
32+
Throwable exception();
33+
34+
/**
35+
* @return the path to the file that the transfer manager failed to upload
36+
*/
37+
Path path();
38+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java

+31-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
2929

3030
/**
31-
* Optional Configurations for the underlying S3 client for which the TransferManager already provides
31+
* Optional configuration for the underlying S3 client for which the TransferManager already provides
3232
* sensible defaults.
3333
*
3434
* <p>Use {@link #builder()} to create a set of options.</p>
@@ -42,6 +42,7 @@ public final class S3ClientConfiguration implements ToCopyableBuilder<S3ClientCo
4242
private final Double targetThroughputInGbps;
4343
private final Integer maxConcurrency;
4444
private final ClientAsyncConfiguration asyncConfiguration;
45+
private final UploadDirectoryConfiguration uploadDirectoryConfiguration;
4546

4647
private S3ClientConfiguration(DefaultBuilder builder) {
4748
this.credentialsProvider = builder.credentialsProvider;
@@ -51,6 +52,7 @@ private S3ClientConfiguration(DefaultBuilder builder) {
5152
this.maxConcurrency = Validate.isPositiveOrNull(builder.maxConcurrency,
5253
"maxConcurrency");
5354
this.asyncConfiguration = builder.asyncConfiguration;
55+
this.uploadDirectoryConfiguration = builder.uploadDirectoryConfiguration;
5456
}
5557

5658
/**
@@ -95,6 +97,13 @@ public Optional<ClientAsyncConfiguration> asyncConfiguration() {
9597
return Optional.ofNullable(asyncConfiguration);
9698
}
9799

100+
/**
101+
* @return the optional upload directory configuration specified
102+
*/
103+
public Optional<UploadDirectoryConfiguration> uploadDirectoryConfiguration() {
104+
return Optional.ofNullable(uploadDirectoryConfiguration);
105+
}
106+
98107
@Override
99108
public Builder toBuilder() {
100109
return new DefaultBuilder(this);
@@ -126,7 +135,10 @@ public boolean equals(Object o) {
126135
if (!Objects.equals(maxConcurrency, that.maxConcurrency)) {
127136
return false;
128137
}
129-
return Objects.equals(asyncConfiguration, that.asyncConfiguration);
138+
if (!Objects.equals(asyncConfiguration, that.asyncConfiguration)) {
139+
return false;
140+
}
141+
return Objects.equals(uploadDirectoryConfiguration, that.uploadDirectoryConfiguration);
130142
}
131143

132144
@Override
@@ -137,6 +149,7 @@ public int hashCode() {
137149
result = 31 * result + (targetThroughputInGbps != null ? targetThroughputInGbps.hashCode() : 0);
138150
result = 31 * result + (maxConcurrency != null ? maxConcurrency.hashCode() : 0);
139151
result = 31 * result + (asyncConfiguration != null ? asyncConfiguration.hashCode() : 0);
152+
result = 31 * result + (uploadDirectoryConfiguration != null ? uploadDirectoryConfiguration.hashCode() : 0);
140153
return result;
141154
}
142155

@@ -253,6 +266,15 @@ public interface Builder extends CopyableBuilder<Builder, S3ClientConfiguration>
253266
default Builder asyncConfiguration(Consumer<ClientAsyncConfiguration.Builder> configuration) {
254267
return asyncConfiguration(ClientAsyncConfiguration.builder().applyMutation(configuration).build());
255268
}
269+
270+
Builder uploadDirectoryConfiguration(UploadDirectoryConfiguration uploadDirectoryConfiguration);
271+
272+
default Builder uploadDirectoryConfiguration(Consumer<UploadDirectoryConfiguration.Builder> uploadConfigurationBuilder) {
273+
Validate.paramNotNull(uploadConfigurationBuilder, "uploadConfigurationBuilder");
274+
return uploadDirectoryConfiguration(UploadDirectoryConfiguration.builder()
275+
.applyMutation(uploadConfigurationBuilder)
276+
.build());
277+
}
256278
}
257279

258280
private static final class DefaultBuilder implements Builder {
@@ -262,6 +284,7 @@ private static final class DefaultBuilder implements Builder {
262284
private Double targetThroughputInGbps;
263285
private Integer maxConcurrency;
264286
private ClientAsyncConfiguration asyncConfiguration;
287+
private UploadDirectoryConfiguration uploadDirectoryConfiguration;
265288

266289
private DefaultBuilder() {
267290
}
@@ -311,6 +334,12 @@ public Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration) {
311334
return this;
312335
}
313336

337+
@Override
338+
public Builder uploadDirectoryConfiguration(UploadDirectoryConfiguration uploadDirectoryConfiguration) {
339+
this.uploadDirectoryConfiguration = uploadDirectoryConfiguration;
340+
return this;
341+
}
342+
314343
@Override
315344
public S3ClientConfiguration build() {
316345
return new S3ClientConfiguration(this);

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java

+50
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,56 @@ default Upload upload(Consumer<UploadRequest.Builder> request) {
160160
return upload(UploadRequest.builder().applyMutation(request).build());
161161
}
162162

163+
/**
164+
* Upload the given directory to the S3 bucket under the given prefix.
165+
*
166+
* <p>
167+
* <b>Usage Example:</b>
168+
* <pre>
169+
* {@code
170+
* UploadDirectory uploadDirectory =
171+
* transferManager.uploadDirectory(UploadDirectoryRequest.builder()
172+
* .sourceDirectory(Paths.get("."))
173+
* .bucket("bucket")
174+
* .prefix("prefix")
175+
* .build());
176+
* // Wait for the transfer to complete
177+
* uploadDirectory.completionFuture().join();
178+
* }
179+
* </pre>
180+
*
181+
* @param uploadDirectoryRequest the upload directory request
182+
* @see #uploadDirectory(UploadDirectoryRequest)
183+
*/
184+
default UploadDirectory uploadDirectory(UploadDirectoryRequest uploadDirectoryRequest) {
185+
throw new UnsupportedOperationException();
186+
}
187+
188+
/**
189+
* Upload the given directory to the S3 bucket under the given prefix.
190+
*
191+
* <p>
192+
* This is a convenience method that creates an instance of the {@link UploadDirectoryRequest} builder avoiding the
193+
* need to create one manually via {@link UploadDirectoryRequest#builder()}.
194+
*
195+
* <p>
196+
* <b>Usage Example:</b>
197+
* <pre>
198+
* {@code
199+
* UploadDirectory uploadDirectory =
200+
* transferManager.uploadDirectory(b -> b.sourceDirectory(Paths.get("."))
201+
* .bucket("key")
202+
* .prefix("prefix"));
203+
* // Wait for the transfer to complete
204+
* uploadDirectory.completionFuture().join();
205+
* }
206+
* </pre>
207+
* @param requestBuilder the upload directory request builder
208+
*/
209+
default UploadDirectory uploadDirectory(Consumer<UploadDirectoryRequest.Builder> requestBuilder) {
210+
return uploadDirectory(UploadDirectoryRequest.builder().applyMutation(requestBuilder).build());
211+
}
212+
163213
/**
164214
* Create an {@code S3TransferManager} using the default values.
165215
*/

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadDirectory.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import java.util.concurrent.CompletableFuture;
19+
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
21+
/**
22+
* An upload transfer of an directory to S3.
23+
*/
24+
@SdkPublicApi
25+
public interface UploadDirectory extends Transfer {
26+
27+
@Override
28+
CompletableFuture<CompletedUploadDirectory> completionFuture();
29+
}

0 commit comments

Comments
 (0)