Skip to content

Commit a4299c7

Browse files
zoewanggzoewangg
committed
Fixed the issue where thresholdInBytes is not the same as minimalPartSizeInBytes if a custom minimalPartSizeInBytes is provided.
1 parent 587a95b commit a4299c7

File tree

4 files changed

+136
-20
lines changed

4 files changed

+136
-20
lines changed

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartConfigurationResolver.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.internal.multipart;
17+
18+
import software.amazon.awssdk.annotations.SdkInternalApi;
19+
import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
20+
import software.amazon.awssdk.utils.Validate;
21+
22+
/**
23+
* Internal utility class to resolve {@link MultipartConfiguration}
24+
*/
25+
@SdkInternalApi
26+
public final class MultipartConfigurationResolver {
27+
28+
private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024;
29+
private final MultipartConfiguration configuration;
30+
31+
public MultipartConfigurationResolver(MultipartConfiguration multipartConfiguration) {
32+
this.configuration = multipartConfiguration;
33+
}
34+
35+
public long resolveMinimalPartSizeInBytes() {
36+
return Validate.getOrDefault(configuration.minimumPartSizeInBytes(), () -> DEFAULT_MIN_PART_SIZE);
37+
}
38+
39+
public long resolveThresholdInBytes() {
40+
return Validate.getOrDefault(configuration.thresholdInBytes(), () -> resolveMinimalPartSizeInBytes());
41+
}
42+
43+
public long resolveApiCallBufferSize() {
44+
return Validate.getOrDefault(configuration.apiCallBufferSizeInBytes(),
45+
() -> resolveMinimalPartSizeInBytes() * 4);
46+
}
47+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,32 +46,21 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient {
4646

4747
private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build();
4848

49-
private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024;
50-
private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024;
51-
private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 4;
52-
5349
private final UploadObjectHelper mpuHelper;
5450
private final CopyObjectHelper copyObjectHelper;
5551

5652
private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) {
5753
super(delegate);
5854
MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration,
5955
MultipartConfiguration.builder()::build);
60-
long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(),
61-
() -> DEFAULT_MIN_PART_SIZE);
62-
long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(),
63-
() -> DEFAULT_THRESHOLD);
64-
long apiCallBufferSizeInBytes = Validate.getOrDefault(validConfiguration.apiCallBufferSizeInBytes(),
65-
() -> computeApiCallBufferSize(validConfiguration));
56+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(validConfiguration);
57+
long minPartSizeInBytes = resolver.resolveMinimalPartSizeInBytes();
58+
long threshold = resolver.resolveThresholdInBytes();
59+
long apiCallBufferSizeInBytes = resolver.resolveApiCallBufferSize();
6660
mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, apiCallBufferSizeInBytes);
6761
copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold);
6862
}
6963

70-
private long computeApiCallBufferSize(MultipartConfiguration multipartConfiguration) {
71-
return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 4
72-
: DEFAULT_API_CALL_BUFFER_SIZE;
73-
}
74-
7564
@Override
7665
public CompletableFuture<PutObjectResponse> putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) {
7766
return mpuHelper.uploadObject(putObjectRequest, requestBody);

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,19 @@ public Long apiCallBufferSizeInBytes() {
9393
public interface Builder extends CopyableBuilder<Builder, MultipartConfiguration> {
9494

9595
/**
96-
* Configures the minimum number of bytes of the body of the request required for requests to be converted to their
97-
* multipart equivalent. Only taken into account when converting {@code putObject} and {@code copyObject} requests.
98-
* Any request whose size is less than the configured value will not use multipart operation,
99-
* even if multipart is enabled via {@link S3AsyncClientBuilder#multipartEnabled(Boolean)}.
96+
* Configure the size threshold, in bytes, for when to use multipart upload. Uploads/copies over this size will
97+
* automatically use a multipart upload strategy, while uploads/copies smaller than this threshold will use a single
98+
* connection to upload/copy the whole object.
99+
*
100100
* <p>
101+
* Multipart uploads are easier to recover from and also potentially faster than single part uploads, especially when the
102+
* upload parts can be uploaded in parallel. Because there are additional network API calls, small objects are still
103+
* recommended to use a single connection for the upload. See
104+
* <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html">Uploading and copying objects using
105+
* multipart upload</a>.
101106
*
102-
* Default value: 8 Mib
107+
* <p>
108+
* By default, it is the same as {@link #minimumPartSizeInBytes(Long)}.
103109
*
104110
* @param thresholdInBytes the value of the threshold to set.
105111
* @return an instance of this builder.

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartConfigurationResolverTest.java

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.internal.multipart;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
20+
import org.junit.jupiter.api.Test;
21+
import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
22+
23+
public class MultipartConfigurationResolverTest {
24+
25+
@Test
26+
void resolveThresholdInBytes_valueNotProvided_shouldSameAsPartSize() {
27+
MultipartConfiguration configuration = MultipartConfiguration.builder()
28+
.minimumPartSizeInBytes(10L)
29+
.build();
30+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
31+
assertThat(resolver.resolveThresholdInBytes()).isEqualTo(10L);
32+
}
33+
34+
@Test
35+
void resolveThresholdInBytes_valueProvided_shouldHonor() {
36+
MultipartConfiguration configuration = MultipartConfiguration.builder()
37+
.minimumPartSizeInBytes(1L)
38+
.thresholdInBytes(12L)
39+
.build();
40+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
41+
assertThat(resolver.resolveThresholdInBytes()).isEqualTo(12L);
42+
}
43+
44+
@Test
45+
void resolveApiCallBufferSize_valueProvided_shouldHonor() {
46+
MultipartConfiguration configuration = MultipartConfiguration.builder()
47+
.apiCallBufferSizeInBytes(100L)
48+
.build();
49+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
50+
assertThat(resolver.resolveApiCallBufferSize()).isEqualTo(100L);
51+
}
52+
53+
@Test
54+
void resolveApiCallBufferSize_valueNotProvided_shouldComputeBasedOnPartSize() {
55+
MultipartConfiguration configuration = MultipartConfiguration.builder()
56+
.minimumPartSizeInBytes(10L)
57+
.build();
58+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
59+
assertThat(resolver.resolveApiCallBufferSize()).isEqualTo(40L);
60+
}
61+
62+
@Test
63+
void valueProvidedForAllFields_shouldHonor() {
64+
MultipartConfiguration configuration = MultipartConfiguration.builder()
65+
.minimumPartSizeInBytes(10L)
66+
.thresholdInBytes(8L)
67+
.apiCallBufferSizeInBytes(3L)
68+
.build();
69+
MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
70+
assertThat(resolver.resolveMinimalPartSizeInBytes()).isEqualTo(10L);
71+
assertThat(resolver.resolveThresholdInBytes()).isEqualTo(8L);
72+
assertThat(resolver.resolveApiCallBufferSize()).isEqualTo(3L);
73+
}
74+
}

0 commit comments

Comments
 (0)