Skip to content

Commit 2e741a8

Browse files
zoewanggzoewangg
committed
Update APIs
1 parent 02922fe commit 2e741a8

File tree

8 files changed

+207
-17
lines changed

8 files changed

+207
-17
lines changed

‎services-custom/s3-transfermanager/pom.xml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,16 @@
6666
<artifactId>annotations</artifactId>
6767
<version>${awsjavasdk.version}</version>
6868
</dependency>
69-
69+
<dependency>
70+
<groupId>software.amazon.awssdk</groupId>
71+
<artifactId>regions</artifactId>
72+
<version>${awsjavasdk.version}</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>software.amazon.awssdk</groupId>
76+
<artifactId>aws-core</artifactId>
77+
<version>${awsjavasdk.version}</version>
78+
</dependency>
7079
<dependency>
7180
<groupId>software.amazon.awssdk</groupId>
7281
<artifactId>service-test-utils</artifactId>

‎services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
import java.nio.file.Path;
1919
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
21+
import software.amazon.awssdk.utils.Validate;
2022
import software.amazon.awssdk.utils.builder.CopyableBuilder;
2123
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
2224

@@ -25,14 +27,17 @@
2527
*/
2628
@SdkPublicApi
2729
public final class DownloadRequest implements TransferRequest, ToCopyableBuilder<DownloadRequest.Builder, DownloadRequest> {
28-
private final String bucket;
29-
private final String key;
3030
private final Path destination;
31+
private final GetObjectRequest getObjectRequest;
3132

3233
private DownloadRequest(BuilderImpl builder) {
33-
this.bucket = builder.bucket;
34-
this.key = builder.key;
34+
Validate.isTrue((builder.bucket != null && builder.key != null) ^ builder.getObjectRequest != null,
35+
"Bucket key pair and the getObjectRequest can't both be configured");
3536
this.destination = builder.destination;
37+
this.getObjectRequest = builder.getObjectRequest == null ? GetObjectRequest.builder()
38+
.bucket(builder.bucket)
39+
.key(builder.key)
40+
.build() : builder.getObjectRequest;
3641
}
3742

3843
/**
@@ -49,13 +54,12 @@ public Builder toBuilder() {
4954

5055
@Override
5156
public String bucket() {
52-
return bucket;
57+
return getObjectRequest.bucket();
5358
}
5459

55-
5660
@Override
5761
public String key() {
58-
return key;
62+
return getObjectRequest.key();
5963
}
6064

6165
/**
@@ -68,7 +72,12 @@ public Path destination() {
6872
return destination;
6973
}
7074

71-
public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builder, DownloadRequest> {
75+
public GetObjectRequest toApiRequest() {
76+
return getObjectRequest;
77+
}
78+
79+
public interface Builder extends TransferRequest.Builder<DownloadRequest, Builder>, CopyableBuilder<Builder,
80+
DownloadRequest> {
7281

7382
/**
7483
* The {@link Path} to file that response contents will be written to. The file must not exist or this method
@@ -79,6 +88,15 @@ public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builde
7988
*/
8089
Builder destination(Path destination);
8190

91+
92+
/**
93+
* The {@link GetObjectRequest} request
94+
*
95+
* @param getObjectRequest the getObject request
96+
* @return a reference to this object so that method calls can be chained together.
97+
*/
98+
Builder apiRequest(GetObjectRequest getObjectRequest);
99+
82100
/**
83101
* @return The built request.
84102
*/
@@ -89,6 +107,7 @@ private static final class BuilderImpl implements Builder {
89107
private String bucket;
90108
private String key;
91109
private Path destination;
110+
private GetObjectRequest getObjectRequest;
92111

93112
private BuilderImpl() {
94113
}
@@ -111,6 +130,12 @@ public Builder key(String key) {
111130
return this;
112131
}
113132

133+
@Override
134+
public Builder apiRequest(GetObjectRequest getObjectRequest) {
135+
this.getObjectRequest = getObjectRequest;
136+
return this;
137+
}
138+
114139
@Override
115140
public DownloadRequest build() {
116141
return new DownloadRequest(this);

‎services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferRequest.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,22 @@ public interface TransferRequest {
3737
*/
3838
String key();
3939

40-
41-
interface Builder {
40+
interface Builder<TypeToBuildT, BuilderT extends Builder> {
4241

4342
/**
4443
* The bucket name containing the object.
4544
*
4645
* @return Returns a reference to this object so that method calls can be chained together.
4746
*/
48-
Builder bucket(String bucket);
47+
BuilderT bucket(String bucket);
4948

5049
/**
5150
* The Key of the object to transfer.
5251
*
5352
* @return Returns a reference to this object so that method calls can be chained together.
5453
*/
55-
Builder key(String key);
54+
BuilderT key(String key);
55+
56+
TypeToBuildT build();
5657
}
5758
}

‎services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public Builder toBuilder() {
6464
}
6565

6666

67-
public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builder, UploadRequest> {
67+
public interface Builder extends TransferRequest.Builder<UploadRequest, Builder>, CopyableBuilder<Builder, UploadRequest> {
6868

6969
/**
7070
* The {@link Path} to file containing data to send to the service. File will be read entirely and may be read

‎services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,31 @@
1515

1616
package software.amazon.awssdk.custom.s3.transfer.internal;
1717

18+
import java.util.ArrayList;
19+
import java.util.List;
1820
import software.amazon.awssdk.annotations.SdkInternalApi;
1921
import software.amazon.awssdk.custom.s3.transfer.S3TransferManager;
2022
import software.amazon.awssdk.services.s3.S3CrtAsyncClient;
23+
import software.amazon.awssdk.utils.SdkAutoCloseable;
2124

2225
@SdkInternalApi
2326
public final class DefaultS3TransferManager implements S3TransferManager {
2427
private final S3CrtAsyncClient s3CrtAsyncClient;
28+
private final List<SdkAutoCloseable> closables = new ArrayList<>();
2529

2630
public DefaultS3TransferManager(DefaultBuilder builder) {
27-
//TODO: create a managed S3CrtAsyncClient if it's not provided
28-
this.s3CrtAsyncClient = builder.s3CrtAsyncClient;
31+
if (builder.s3CrtAsyncClient == null) {
32+
s3CrtAsyncClient = S3CrtAsyncClient.builder()
33+
.build();
34+
closables.add(s3CrtAsyncClient);
35+
} else {
36+
s3CrtAsyncClient = builder.s3CrtAsyncClient;
37+
}
2938
}
3039

3140
@Override
3241
public void close() {
33-
s3CrtAsyncClient.close();
42+
closables.forEach(SdkAutoCloseable::close);
3443
}
3544

3645
public static Builder builder() {

‎services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.custom.s3.transfer.util;
17+
18+
import org.junit.BeforeClass;
19+
import software.amazon.awssdk.regions.Region;
20+
import software.amazon.awssdk.services.s3.S3AsyncClient;
21+
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
22+
import software.amazon.awssdk.services.s3.S3Client;
23+
import software.amazon.awssdk.services.s3.S3ClientBuilder;
24+
import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
25+
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
26+
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
27+
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
28+
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
29+
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
30+
import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
31+
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
32+
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
33+
import software.amazon.awssdk.services.s3.model.ObjectVersion;
34+
import software.amazon.awssdk.services.s3.model.S3Exception;
35+
import software.amazon.awssdk.services.s3.model.S3Object;
36+
import software.amazon.awssdk.testutils.service.AwsTestBase;
37+
38+
/**
39+
* Base class for S3 integration tests. Loads AWS credentials from a properties
40+
* file and creates an S3 client for callers to use.
41+
*/
42+
public class S3IntegrationTestBase extends AwsTestBase {
43+
44+
protected static final Region DEFAULT_REGION = Region.US_WEST_2;
45+
/**
46+
* The S3 client for all tests to use.
47+
*/
48+
protected static S3Client s3;
49+
50+
protected static S3AsyncClient s3Async;
51+
52+
/**
53+
* Loads the AWS account info for the integration tests and creates an S3
54+
* client for tests to use.
55+
*/
56+
@BeforeClass
57+
public static void setUp() throws Exception {
58+
s3 = s3ClientBuilder().build();
59+
s3Async = s3AsyncClientBuilder().build();
60+
}
61+
62+
protected static S3ClientBuilder s3ClientBuilder() {
63+
return S3Client.builder()
64+
.region(DEFAULT_REGION)
65+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
66+
}
67+
68+
protected static S3AsyncClientBuilder s3AsyncClientBuilder() {
69+
return S3AsyncClient.builder()
70+
.region(DEFAULT_REGION)
71+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
72+
}
73+
74+
protected static void createBucket(String bucketName) {
75+
createBucket(bucketName, 0);
76+
}
77+
78+
private static void createBucket(String bucketName, int retryCount) {
79+
try {
80+
s3.createBucket(
81+
CreateBucketRequest.builder()
82+
.bucket(bucketName)
83+
.createBucketConfiguration(
84+
CreateBucketConfiguration.builder()
85+
.locationConstraint(BucketLocationConstraint.US_WEST_2)
86+
.build())
87+
.build());
88+
} catch (S3Exception e) {
89+
System.err.println("Error attempting to create bucket: " + bucketName);
90+
if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) {
91+
System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName);
92+
} else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) {
93+
System.err.println("Printing all buckets for debug:");
94+
s3.listBuckets().buckets().forEach(System.err::println);
95+
if (retryCount < 2) {
96+
System.err.println("Retrying...");
97+
createBucket(bucketName, retryCount + 1);
98+
} else {
99+
throw e;
100+
}
101+
} else {
102+
throw e;
103+
}
104+
}
105+
}
106+
107+
protected static void deleteBucketAndAllContents(String bucketName) {
108+
System.out.println("Deleting S3 bucket: " + bucketName);
109+
ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
110+
111+
while (true) {
112+
if (response.contents() == null) {
113+
break;
114+
}
115+
for (S3Object objectSummary : response.contents()) {
116+
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
117+
}
118+
119+
if (response.isTruncated()) {
120+
response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
121+
} else {
122+
break;
123+
}
124+
}
125+
126+
ListObjectVersionsResponse versionsResponse = s3
127+
.listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
128+
if (versionsResponse.versions() != null) {
129+
for (ObjectVersion s : versionsResponse.versions()) {
130+
s3.deleteObject(DeleteObjectRequest.builder()
131+
.bucket(bucketName)
132+
.key(s.key())
133+
.versionId(s.versionId())
134+
.build());
135+
}
136+
}
137+
138+
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
139+
}
140+
141+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,7 @@
2727
@SdkPublicApi
2828
public interface S3CrtAsyncClientBuilder extends AwsClientBuilder<S3CrtAsyncClientBuilder, S3CrtAsyncClient> {
2929

30+
S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes);
31+
32+
S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps);
3033
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,13 @@ public S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride) {
7777
throw new UnsupportedOperationException();
7878
}
7979

80+
@Override
8081
public S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes) {
8182
this.partSizeBytes = partSizeBytes;
8283
return this;
8384
}
8485

86+
@Override
8587
public S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps) {
8688
this.maxThroughputGbps = maxThroughputGbps;
8789
return this;

0 commit comments

Comments
 (0)