Skip to content

Commit e0f7c0e

Browse files
dagnirzoewangg
dagnir
authored and
zoewangg
committed
Add download implementation for API requests
1 parent 16b1f57 commit e0f7c0e

File tree

53 files changed

+4568
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+4568
-6
lines changed

‎.changes/next-release/bugfix-AWSSDKforJavav2-f0e1099.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"category": "AWS SDK for Java v2",
3+
"type": "bugfix",
4+
"description": "Fix a bug where events in an event stream were being signed with the request date, and not with the current system time."
5+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,26 @@ static <ResponseT> AsyncResponseTransformer<ResponseT, ResponseT> toFile(Path pa
119119
return new FileAsyncResponseTransformer<>(path);
120120
}
121121

122+
/**
123+
* Creates an {@link AsyncResponseTransformer} that writes all the content to the given file.
124+
*
125+
* @param path Path to file to write to.
126+
* @param position The value for the data between the current end of the file and the starting position is
127+
* undefined.
128+
* @param isNewFile Whether this is a new file. If this is {@code true} and the file already exists, the
129+
* transformer will complete with an exception.
130+
* @param deleteOnFailure Whether the file on disk should be deleted in the event of a failure when writing the
131+
* stream.
132+
* @param <ResponseT> Pojo Response type.
133+
* @return AsyncResponseTransformer instance.
134+
*/
135+
static <ResponseT> AsyncResponseTransformer<ResponseT, ResponseT> toFile(Path path,
136+
long position,
137+
boolean isNewFile,
138+
boolean deleteOnFailure) {
139+
return new FileAsyncResponseTransformer<>(path, position, isNewFile, deleteOnFailure);
140+
}
141+
122142
/**
123143
* Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error,
124144
* the SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an

‎core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,4 @@ public void cancel() {
128128
});
129129
}
130130
}
131-
}
131+
}

‎docs/design/services/s3/transfermanager/prototype.java

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,16 @@ interface Builder {
221221
*/
222222
Builder maxDownloadBytesPerSecond(Long maxDownloadBytesPerSecond);
223223

224+
/**
225+
* The multipart download configuration.
226+
*/
227+
Builder multipartDownloadConfiguration(MultipartDownloadConfiguration multipartDownloadConfiguration);
228+
229+
/**
230+
* The multipart upload configuration.
231+
*/
232+
Builder multipartUploadConfiguration(MultipartUploadConfiguration multipartUploadConfiguration);
233+
224234
/**
225235
* Add a progress listener to the currently configured list of
226236
* listeners.
@@ -445,11 +455,6 @@ public interface SinglePartDownloadContext {
445455
* The original download request given to the Transfer Manager.
446456
*/
447457
DownloadObjectRequest downloadRequest();
448-
449-
/**
450-
* The request sent to S3 for this object. This is empty if downloading a presigned URL.
451-
*/
452-
GetObjectRequest objectRequest();
453458
}
454459

455460
/**

‎services-custom/pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
<modules>
3131
<module>dynamodb-enhanced</module>
32+
<module>s3-transfermanager</module>
3233
</modules>
3334

3435
<dependencyManagement>

‎services-custom/s3-transfermanager/pom.xml

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
~ Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License").
6+
~ You may not use this file except in compliance with the License.
7+
~ A copy of the License is located at
8+
~
9+
~ http://aws.amazon.com/apache2.0
10+
~
11+
~ or in the "license" file accompanying this file. This file is distributed
12+
~ on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
13+
~ express or implied. See the License for the specific language governing
14+
~ permissions and limitations under the License.
15+
-->
16+
17+
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
18+
xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>software.amazon.awssdk</groupId>
23+
<artifactId>aws-sdk-java-pom</artifactId>
24+
<version>2.5.32-SNAPSHOT</version>
25+
<relativePath>../../pom.xml</relativePath>
26+
</parent>
27+
<artifactId>s3-transfermanager</artifactId>
28+
<version>preview-SNAPSHOT</version>
29+
<name>AWS Java SDK :: S3 :: Transfer Manager</name>
30+
<description>
31+
The S3 Transfer Manager allows customers to easily and optimally
32+
transfer objects and directories to and from S3.
33+
</description>
34+
<url>https://aws.amazon.com/sdkforjava</url>
35+
36+
<properties>
37+
<jre.version>1.8</jre.version>
38+
</properties>
39+
40+
<dependencyManagement>
41+
<dependencies>
42+
<dependency>
43+
<groupId>software.amazon.awssdk</groupId>
44+
<artifactId>bom-internal</artifactId>
45+
<version>2.5.32-SNAPSHOT</version>
46+
<type>pom</type>
47+
<scope>import</scope>
48+
</dependency>
49+
</dependencies>
50+
</dependencyManagement>
51+
52+
<dependencies>
53+
<dependency>
54+
<groupId>software.amazon.awssdk</groupId>
55+
<artifactId>s3</artifactId>
56+
<version>2.5.32-SNAPSHOT</version>
57+
</dependency>
58+
<dependency>
59+
<groupId>software.amazon.awssdk</groupId>
60+
<artifactId>sdk-core</artifactId>
61+
<version>2.5.32-SNAPSHOT</version>
62+
</dependency>
63+
<dependency>
64+
<groupId>software.amazon.awssdk</groupId>
65+
<artifactId>aws-core</artifactId>
66+
<version>2.5.32-SNAPSHOT</version>
67+
</dependency>
68+
<dependency>
69+
<groupId>software.amazon.awssdk</groupId>
70+
<artifactId>regions</artifactId>
71+
<version>2.5.32-SNAPSHOT</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>software.amazon.awssdk</groupId>
75+
<artifactId>utils</artifactId>
76+
<version>2.5.32-SNAPSHOT</version>
77+
</dependency>
78+
<dependency>
79+
<groupId>software.amazon.awssdk</groupId>
80+
<artifactId>annotations</artifactId>
81+
<version>2.5.32-SNAPSHOT</version>
82+
</dependency>
83+
84+
<dependency>
85+
<groupId>software.amazon.awssdk</groupId>
86+
<artifactId>service-test-utils</artifactId>
87+
<version>2.5.32-SNAPSHOT</version>
88+
<scope>test</scope>
89+
</dependency>
90+
<dependency>
91+
<groupId>software.amazon.awssdk</groupId>
92+
<artifactId>test-utils</artifactId>
93+
<version>2.5.32-SNAPSHOT</version>
94+
<scope>test</scope>
95+
</dependency>
96+
<dependency>
97+
<artifactId>junit</artifactId>
98+
<groupId>junit</groupId>
99+
<scope>test</scope>
100+
</dependency>
101+
<dependency>
102+
<groupId>org.assertj</groupId>
103+
<artifactId>assertj-core</artifactId>
104+
<scope>test</scope>
105+
</dependency>
106+
<dependency>
107+
<groupId>org.mockito</groupId>
108+
<artifactId>mockito-core</artifactId>
109+
<scope>test</scope>
110+
</dependency>
111+
<dependency>
112+
<groupId>log4j</groupId>
113+
<artifactId>log4j</artifactId>
114+
<scope>test</scope>
115+
</dependency>
116+
<dependency>
117+
<groupId>org.slf4j</groupId>
118+
<artifactId>slf4j-log4j12</artifactId>
119+
<scope>test</scope>
120+
</dependency>
121+
</dependencies>
122+
</project>

‎services-custom/s3-transfermanager/src/it/java/software/amazon/awssdk/custom/s3/transfer/DownloadIntegrationTest.java

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
* Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.custom.s3.transfer;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import java.io.IOException;
20+
import java.io.InputStream;
21+
import java.io.UncheckedIOException;
22+
import java.nio.file.Files;
23+
import java.nio.file.Path;
24+
import java.nio.file.Paths;
25+
import java.nio.file.StandardOpenOption;
26+
import java.security.MessageDigest;
27+
import java.security.NoSuchAlgorithmException;
28+
import org.junit.AfterClass;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
import software.amazon.awssdk.custom.s3.transfer.util.SizeConstant;
32+
import software.amazon.awssdk.testutils.RandomTempFile;
33+
import software.amazon.awssdk.testutils.service.S3BucketUtils;
34+
import software.amazon.awssdk.utils.BinaryUtils;
35+
36+
/**
37+
* Integration test for TransferManager downloads.
38+
*/
39+
public class DownloadIntegrationTest extends S3TransferManagerIntegrationTestBase {
40+
private static final String BUCKET = S3BucketUtils.temporaryBucketName(DownloadIntegrationTest.class);
41+
private static final String KEY_8KiB = "8kb_test_file.dat";
42+
private static final String KEY_16MiB = "16mb_test_file.dat";
43+
private static final Path TMP_DIR = Paths.get(System.getProperty("java.io.tmpdir"));
44+
private static final MessageDigest MD5_DIGEST;
45+
46+
static {
47+
try {
48+
MD5_DIGEST = MessageDigest.getInstance("MD5");
49+
} catch (NoSuchAlgorithmException e) {
50+
throw new RuntimeException("Could not instantiate MD5 digest");
51+
}
52+
}
53+
54+
private static S3TransferManager transferManager;
55+
56+
private static Path testFile8KiB;
57+
private static Path testFile16MiB;
58+
59+
private static String testFile8KiBDigest;
60+
private static String testFile16MiBDigest;
61+
62+
@BeforeClass
63+
public static void setup() throws Exception {
64+
S3TransferManagerIntegrationTestBase.setUp();
65+
66+
transferManager = S3TransferManager.builder()
67+
.s3client(s3Async)
68+
.multipartDownloadConfiguration(MultipartDownloadConfiguration.defaultConfig()
69+
.toBuilder()
70+
.multipartDownloadThreshold(16 * SizeConstant.MiB - 1)
71+
.build())
72+
.build();
73+
74+
testFile8KiB = new RandomTempFile(8 * SizeConstant.KiB).toPath();
75+
testFile16MiB = new RandomTempFile(16 * SizeConstant.MiB).toPath();
76+
77+
testFile8KiBDigest = computeMd5(testFile8KiB);
78+
testFile16MiBDigest = computeMd5(testFile16MiB);
79+
80+
createBucket(BUCKET);
81+
putFile(KEY_8KiB, testFile8KiB);
82+
putFile(KEY_16MiB, testFile16MiB);
83+
}
84+
85+
@AfterClass
86+
public static void teardown() {
87+
deleteBucketAndAllContents(BUCKET);
88+
tryDeleteFiles(testFile8KiB, testFile16MiB);
89+
}
90+
91+
@Test
92+
public void singlePartDownload() throws IOException {
93+
downloadTest(KEY_8KiB, testFile8KiBDigest);
94+
}
95+
96+
@Test
97+
public void multipartDownload() throws IOException {
98+
downloadTest(KEY_16MiB, testFile16MiBDigest);
99+
}
100+
101+
private static void putFile(String key, Path file) {
102+
s3.putObject(r -> r.bucket(BUCKET).key(key), file);
103+
}
104+
105+
private static void downloadTest(String key, String expectedMd5) throws IOException {
106+
Path tempFile = createTempPath();
107+
try {
108+
transferManager.download(BUCKET, key, tempFile).completionFuture().join();
109+
String downloadedFileMd5 = computeMd5(tempFile);
110+
assertThat(downloadedFileMd5).isEqualTo(expectedMd5);
111+
} finally {
112+
Files.deleteIfExists(tempFile);
113+
}
114+
}
115+
116+
private static Path createTempPath() {
117+
return TMP_DIR.resolve(DownloadIntegrationTest.class.getSimpleName() + "-" + System.currentTimeMillis());
118+
}
119+
120+
private static String computeMd5(Path file) {
121+
try (InputStream is = Files.newInputStream(file, StandardOpenOption.READ)) {
122+
MD5_DIGEST.reset();
123+
byte[] buff = new byte[4096];
124+
int read;
125+
while ((read = is.read(buff)) != -1) {
126+
MD5_DIGEST.update(buff, 0, read);
127+
}
128+
return BinaryUtils.toBase64(MD5_DIGEST.digest());
129+
} catch (IOException e) {
130+
throw new UncheckedIOException(e);
131+
}
132+
}
133+
134+
private static void tryDeleteFiles(Path... files) {
135+
for (Path file : files) {
136+
try {
137+
Files.deleteIfExists(file);
138+
} catch (IOException e) {
139+
System.err.println("Could not delete file " + file);
140+
}
141+
}
142+
}
143+
}

0 commit comments

Comments
 (0)