Skip to content

Commit 4a6f5c4

Browse files
zoewanggzoewangg
authored
Support upload directory in s3 transfer manager (#2743)
* Support upload directory in s3 transfer manager
1 parent 696e95d commit 4a6f5c4

37 files changed

+3409
-284
lines changed

‎pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@
9898
<slf4j.version>1.7.30</slf4j.version>
9999
<log4j.version>1.2.17</log4j.version>
100100
<commons.io.version>2.5</commons.io.version>
101-
<equalsverifier.version>3.5</equalsverifier.version>
101+
<equalsverifier.version>3.7.1</equalsverifier.version>
102102
<!-- Update netty-open-ssl-version accordingly whenever we update netty version-->
103103
<!-- https://github.com/netty/netty/blob/4.1/pom.xml search "tcnative.version" -->
104104
<netty.version>4.1.68.Final</netty.version>

‎services-custom/s3-transfer-manager/pom.xml

+15
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@
8686
<artifactId>http-client-spi</artifactId>
8787
<version>${awsjavasdk.version}</version>
8888
</dependency>
89+
<dependency>
90+
<groupId>software.amazon.awssdk</groupId>
91+
<artifactId>arns</artifactId>
92+
<version>${awsjavasdk.version}</version>
93+
</dependency>
8994
<dependency>
9095
<groupId>software.amazon.awssdk</groupId>
9196
<artifactId>auth</artifactId>
@@ -153,6 +158,16 @@
153158
<artifactId>reactive-streams-tck</artifactId>
154159
<scope>test</scope>
155160
</dependency>
161+
<dependency>
162+
<groupId>com.google.jimfs</groupId>
163+
<artifactId>jimfs</artifactId>
164+
<scope>test</scope>
165+
</dependency>
166+
<dependency>
167+
<groupId>nl.jqno.equalsverifier</groupId>
168+
<artifactId>equalsverifier</artifactId>
169+
<scope>test</scope>
170+
</dependency>
156171
</dependencies>
157172

158173
<build>

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadDirectoryIntegrationTest.java

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
21+
22+
import java.io.IOException;
23+
import java.nio.charset.StandardCharsets;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.nio.file.Paths;
27+
import java.util.List;
28+
import java.util.stream.Collectors;
29+
import org.apache.commons.lang3.RandomStringUtils;
30+
import org.junit.AfterClass;
31+
import org.junit.BeforeClass;
32+
import org.junit.Test;
33+
import software.amazon.awssdk.core.sync.ResponseTransformer;
34+
import software.amazon.awssdk.services.s3.S3Client;
35+
import software.amazon.awssdk.services.s3.model.S3Object;
36+
import software.amazon.awssdk.testutils.FileUtils;
37+
import software.amazon.awssdk.utils.Logger;
38+
39+
public class S3TransferManagerUploadDirectoryIntegrationTest extends S3IntegrationTestBase {
40+
private static final Logger log = Logger.loggerFor(S3TransferManagerUploadDirectoryIntegrationTest.class);
41+
private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
42+
43+
private static S3TransferManager tm;
44+
private static Path directory;
45+
private static S3Client s3Client;
46+
private static String randomString;
47+
48+
@BeforeClass
49+
public static void setUp() throws Exception {
50+
S3IntegrationTestBase.setUp();
51+
createBucket(TEST_BUCKET);
52+
randomString = RandomStringUtils.random(100);
53+
directory = createLocalTestDirectory();
54+
55+
tm = S3TransferManager.builder()
56+
.s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
57+
.region(DEFAULT_REGION)
58+
.maxConcurrency(100))
59+
.build();
60+
61+
s3Client = S3Client.builder()
62+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN).region(DEFAULT_REGION)
63+
.build();
64+
}
65+
66+
@AfterClass
67+
public static void teardown() {
68+
try {
69+
FileUtils.cleanUpTestDirectory(directory);
70+
} catch (Exception exception) {
71+
log.warn(() -> "Failed to clean up test directory " + directory, exception);
72+
}
73+
74+
try {
75+
deleteBucketAndAllContents(TEST_BUCKET);
76+
} catch (Exception exception) {
77+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET, exception);
78+
}
79+
80+
closeQuietly(tm, log.logger());
81+
closeQuietly(s3Client, log.logger());
82+
S3IntegrationTestBase.cleanUp();
83+
}
84+
85+
@Test
86+
public void uploadDirectory_filesSentCorrectly() {
87+
String prefix = "yolo";
88+
UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
89+
.bucket(TEST_BUCKET)
90+
.prefix(prefix)
91+
.overrideConfiguration(o -> o.recursive(true)));
92+
CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
93+
assertThat(completedUploadDirectory.failedUploads()).isEmpty();
94+
95+
List<String> keys =
96+
s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
97+
.collect(Collectors.toList());
98+
99+
assertThat(keys).containsOnly(prefix + "/bar.txt", prefix + "/foo/1.txt", prefix + "/foo/2.txt");
100+
101+
keys.forEach(k -> verifyContent(k, k.substring(prefix.length() + 1) + randomString));
102+
}
103+
104+
@Test
105+
public void uploadDirectory_withDelimiter_filesSentCorrectly() {
106+
String prefix = "hello";
107+
String delimiter = "0";
108+
UploadDirectoryTransfer uploadDirectory = tm.uploadDirectory(u -> u.sourceDirectory(directory)
109+
.bucket(TEST_BUCKET)
110+
.delimiter(delimiter)
111+
.prefix(prefix)
112+
.overrideConfiguration(o -> o.recursive(true)));
113+
CompletedUploadDirectory completedUploadDirectory = uploadDirectory.completionFuture().join();
114+
assertThat(completedUploadDirectory.failedUploads()).isEmpty();
115+
116+
List<String> keys =
117+
s3Client.listObjectsV2Paginator(b -> b.bucket(TEST_BUCKET).prefix(prefix)).contents().stream().map(S3Object::key)
118+
.collect(Collectors.toList());
119+
120+
assertThat(keys).containsOnly(prefix + "0bar.txt", prefix + "0foo01.txt", prefix + "0foo02.txt");
121+
keys.forEach(k -> {
122+
String path = k.replace(delimiter, "/");
123+
verifyContent(k, path.substring(prefix.length() + 1) + randomString);
124+
});
125+
}
126+
127+
private static Path createLocalTestDirectory() throws IOException {
128+
Path directory = Files.createTempDirectory("test");
129+
130+
String directoryName = directory.toString();
131+
132+
Files.createDirectory(Paths.get(directory + "/foo"));
133+
Files.write(Paths.get(directoryName, "bar.txt"), ("bar.txt" + randomString).getBytes(StandardCharsets.UTF_8));
134+
Files.write(Paths.get(directoryName, "foo/1.txt"), ("foo/1.txt" + randomString).getBytes(StandardCharsets.UTF_8));
135+
Files.write(Paths.get(directoryName, "foo/2.txt"), ("foo/2.txt" + randomString).getBytes(StandardCharsets.UTF_8));
136+
137+
return directory;
138+
}
139+
140+
private static void verifyContent(String key, String expectedContent) {
141+
String actualContent = s3.getObject(r -> r.bucket(TEST_BUCKET).key(key),
142+
ResponseTransformer.toBytes()).asUtf8String();
143+
144+
assertThat(actualContent).isEqualTo(expectedContent);
145+
}
146+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java

+80-3
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,95 @@
1717

1818
import software.amazon.awssdk.annotations.SdkPreviewApi;
1919
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
import software.amazon.awssdk.services.s3.S3AsyncClient;
2021
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
22+
import software.amazon.awssdk.utils.Validate;
2123

2224
/**
23-
* A completed download transfer.
25+
* Represents a completed download transfer from Amazon S3. It can be used to track
26+
* the underlying {@link GetObjectResponse}
27+
*
28+
* @see S3TransferManager#download(DownloadRequest)
2429
*/
2530
@SdkPublicApi
2631
@SdkPreviewApi
27-
public interface CompletedDownload extends CompletedTransfer {
32+
public final class CompletedDownload implements CompletedTransfer {
33+
private final GetObjectResponse response;
34+
35+
private CompletedDownload(DefaultBuilder builder) {
36+
this.response = Validate.paramNotNull(builder.response, "response");
37+
}
2838

2939
/**
3040
* Returns the API response from the {@link S3TransferManager#download(DownloadRequest)}
3141
* @return the response
3242
*/
33-
GetObjectResponse response();
43+
public GetObjectResponse response() {
44+
return response;
45+
}
46+
47+
@Override
48+
public boolean equals(Object o) {
49+
if (this == o) {
50+
return true;
51+
}
52+
if (o == null || getClass() != o.getClass()) {
53+
return false;
54+
}
55+
56+
CompletedDownload that = (CompletedDownload) o;
57+
58+
return response.equals(that.response);
59+
}
60+
61+
@Override
62+
public int hashCode() {
63+
return response.hashCode();
64+
}
65+
66+
public static Builder builder() {
67+
return new DefaultBuilder();
68+
}
69+
70+
public interface Builder {
71+
/**
72+
* Specify the {@link GetObjectResponse} from {@link S3AsyncClient#getObject}
73+
*
74+
* @param response the response
75+
* @return This builder for method chaining.
76+
*/
77+
Builder response(GetObjectResponse response);
78+
79+
/**
80+
* Builds a {@link CompletedUpload} based on the properties supplied to this builder
81+
* @return An initialized {@link CompletedUpload}
82+
*/
83+
CompletedDownload build();
84+
}
85+
86+
private static final class DefaultBuilder implements Builder {
87+
private GetObjectResponse response;
88+
89+
private DefaultBuilder() {
90+
}
91+
92+
@Override
93+
public Builder response(GetObjectResponse response) {
94+
this.response = response;
95+
return this;
96+
}
97+
98+
public void setResponse(GetObjectResponse response) {
99+
response(response);
100+
}
101+
102+
public GetObjectResponse getResponse() {
103+
return response;
104+
}
105+
106+
@Override
107+
public CompletedDownload build() {
108+
return new CompletedDownload(this);
109+
}
110+
}
34111
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java

+87-3
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,102 @@
1717

1818
import software.amazon.awssdk.annotations.SdkPreviewApi;
1919
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
import software.amazon.awssdk.services.s3.S3AsyncClient;
2021
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
22+
import software.amazon.awssdk.utils.ToString;
23+
import software.amazon.awssdk.utils.Validate;
2124

2225
/**
23-
* A completed upload transfer.
26+
* Represents a completed upload transfer to Amazon S3. It can be used to track
27+
* the underlying {@link PutObjectResponse}
28+
*
29+
* @see S3TransferManager#upload(UploadRequest)
2430
*/
2531
@SdkPublicApi
2632
@SdkPreviewApi
27-
public interface CompletedUpload extends CompletedTransfer {
33+
public final class CompletedUpload implements CompletedTransfer {
34+
private final PutObjectResponse response;
35+
36+
private CompletedUpload(DefaultBuilder builder) {
37+
this.response = Validate.paramNotNull(builder.response, "response");
38+
}
2839

2940
/**
3041
* Returns the API response from the {@link S3TransferManager#upload(UploadRequest)}
3142
* @return the response
3243
*/
33-
PutObjectResponse response();
44+
public PutObjectResponse response() {
45+
return response;
46+
}
47+
48+
@Override
49+
public boolean equals(Object o) {
50+
if (this == o) {
51+
return true;
52+
}
53+
if (o == null || getClass() != o.getClass()) {
54+
return false;
55+
}
56+
57+
CompletedUpload that = (CompletedUpload) o;
58+
59+
return response.equals(that.response);
60+
}
61+
62+
@Override
63+
public int hashCode() {
64+
return response.hashCode();
65+
}
66+
67+
@Override
68+
public String toString() {
69+
return ToString.builder("CompletedUpload")
70+
.add("response", response)
71+
.build();
72+
}
73+
74+
public static Class<? extends Builder> serializableBuilderClass() {
75+
return DefaultBuilder.class;
76+
}
77+
78+
/**
79+
* Creates a default builder for {@link CompletedUpload}.
80+
*/
81+
public static Builder builder() {
82+
return new DefaultBuilder();
83+
}
84+
85+
public interface Builder {
86+
/**
87+
* Specify the {@link PutObjectResponse} from {@link S3AsyncClient#putObject}
88+
*
89+
* @param response the response
90+
* @return This builder for method chaining.
91+
*/
92+
Builder response(PutObjectResponse response);
93+
94+
/**
95+
* Builds a {@link CompletedUpload} based on the properties supplied to this builder
96+
* @return An initialized {@link CompletedUpload}
97+
*/
98+
CompletedUpload build();
99+
}
100+
101+
private static class DefaultBuilder implements Builder {
102+
private PutObjectResponse response;
103+
104+
private DefaultBuilder() {
105+
}
106+
107+
@Override
108+
public Builder response(PutObjectResponse response) {
109+
this.response = response;
110+
return this;
111+
}
112+
113+
@Override
114+
public CompletedUpload build() {
115+
return new CompletedUpload(this);
116+
}
117+
}
34118
}

0 commit comments

Comments
 (0)