From 2e0a841a7d498315618122e9e1c5ea813575495a Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Jul 2023 16:54:10 -0400 Subject: [PATCH 01/10] Multipart API fix merge conflicts --- .../customization/CustomizationConfig.java | 13 ++ .../customization/MultipartCustomization.java | 46 ++++ .../config/customization/ServiceConfig.java | 16 ++ .../amazon/awssdk/codegen/poet/ClassSpec.java | 2 +- .../poet/builder/AsyncClientBuilderClass.java | 66 ++++-- .../builder/AsyncClientBuilderInterface.java | 77 ++++++- .../services/s3/S3IntegrationTestBase.java | 2 +- .../S3ClientMultiPartCopyIntegrationTest.java | 12 +- ...ltipartClientPutObjectIntegrationTest.java | 25 ++- .../client/S3AsyncClientDecorator.java | 23 +- .../internal/multipart/CopyObjectHelper.java | 28 ++- .../multipart/GenericMultipartHelper.java | 11 +- .../multipart/MultipartS3AsyncClient.java | 27 ++- .../multipart/MultipartUploadHelper.java | 13 +- .../s3/multipart/MultipartConfiguration.java | 206 ++++++++++++++++++ .../codegen-resources/customization.config | 5 + .../MultipartClientUserAgentTest.java | 78 +++++++ .../S3MultipartClientBuilderTest.java | 63 ++++++ 18 files changed, 662 insertions(+), 51 deletions(-) create mode 100644 codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index 596d44bcf14b..0bef67df7867 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -227,6 +227,11 @@ public class CustomizationConfig { */ private String asyncClientDecorator; + /** + * Only for s3. A set of customization to related to multipart operations. + */ + private MultipartCustomization multipartCustomization; + /** * Whether to skip generating endpoint tests from endpoint-tests.json */ @@ -665,4 +670,12 @@ public Map getCustomClientContextParams() { public void setCustomClientContextParams(Map customClientContextParams) { this.customClientContextParams = customClientContextParams; } + + public MultipartCustomization getMultipartCustomization() { + return this.multipartCustomization; + } + + public void setMultipartCustomization(MultipartCustomization multipartCustomization) { + this.multipartCustomization = multipartCustomization; + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java new file mode 100644 index 000000000000..6f87cc50ce6c --- /dev/null +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java @@ -0,0 +1,46 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.model.config.customization; + +public class MultipartCustomization { + private String multipartConfigurationClass; + private String multipartConfigMethodDoc; + private String multipartEnableMethodDoc; + + public String getMultipartConfigurationClass() { + return multipartConfigurationClass; + } + + public void setMultipartConfigurationClass(String multipartConfigurationClass) { + this.multipartConfigurationClass = multipartConfigurationClass; + } + + public String getMultipartConfigMethodDoc() { + return multipartConfigMethodDoc; + } + + public void setMultipartConfigMethodDoc(String multipartMethodDoc) { + this.multipartConfigMethodDoc = multipartMethodDoc; + } + + public String getMultipartEnableMethodDoc() { + return multipartEnableMethodDoc; + } + + public void setMultipartEnableMethodDoc(String multipartEnableMethodDoc) { + this.multipartEnableMethodDoc = multipartEnableMethodDoc; + } +} diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java index b0c0db862df2..3dc4b6291b5e 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.codegen.model.config.customization; +import software.amazon.awssdk.utils.ToString; + public class ServiceConfig { /** * Specifies the name of the client configuration class to use if a service @@ -112,4 +114,18 @@ public boolean hasAccelerateModeEnabledProperty() { public void setHasAccelerateModeEnabledProperty(boolean hasAccelerateModeEnabledProperty) { this.hasAccelerateModeEnabledProperty = hasAccelerateModeEnabledProperty; } + + @Override + public String toString() { + return ToString.builder("ServiceConfig") + .add("className", className) + .add("hasDualstackProperty", hasDualstackProperty) + .add("hasFipsProperty", hasFipsProperty) + .add("hasUseArnRegionProperty", hasUseArnRegionProperty) + .add("hasMultiRegionEnabledProperty", hasMultiRegionEnabledProperty) + .add("hasPathStyleAccessEnabledProperty", hasPathStyleAccessEnabledProperty) + .add("hasAccelerateModeEnabledProperty", hasAccelerateModeEnabledProperty) + .add("hasCrossRegionAccessEnabledProperty", hasCrossRegionAccessEnabledProperty) + .build(); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java index a8265f0dc7f1..59a719fb2c7d 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java @@ -20,7 +20,7 @@ import java.util.Collections; /** - * Represents the a Poet generated class + * Represents a Poet generated class */ public interface ClassSpec { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java index 509a30c6c8d7..efe81f2cb9a1 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java @@ -17,6 +17,7 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; import java.net.URI; @@ -24,6 +25,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.auth.token.credentials.SdkTokenProvider; import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetExtension; @@ -59,12 +61,12 @@ public AsyncClientBuilderClass(IntermediateModel model) { @Override public TypeSpec poetSpec() { TypeSpec.Builder builder = - PoetUtils.createClassBuilder(builderClassName) - .addAnnotation(SdkInternalApi.class) - .addModifiers(Modifier.FINAL) - .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) - .addSuperinterface(builderInterfaceName) - .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); + PoetUtils.createClassBuilder(builderClassName) + .addAnnotation(SdkInternalApi.class) + .addModifiers(Modifier.FINAL) + .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) + .addSuperinterface(builderInterfaceName) + .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); if (model.getEndpointOperation().isPresent()) { builder.addMethod(endpointDiscoveryEnabled()); @@ -80,6 +82,12 @@ public TypeSpec poetSpec() { builder.addMethod(bearerTokenProviderMethod()); } + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + builder.addMethod(multipartEnabledMethod(multipartCustomization)); + builder.addMethod(multipartConfigMethods(multipartCustomization)); + } + builder.addMethod(buildClientMethod()); builder.addMethod(initializeServiceClientConfigMethod()); @@ -124,15 +132,15 @@ private MethodSpec endpointProviderMethod() { private MethodSpec buildClientMethod() { MethodSpec.Builder builder = MethodSpec.methodBuilder("buildClient") - .addAnnotation(Override.class) - .addModifiers(Modifier.PROTECTED, Modifier.FINAL) - .returns(clientInterfaceName) - .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", - SdkClientConfiguration.class).addStatement("this.validateClientOptions" - + "(clientConfiguration)") - .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" - + "(clientConfiguration)", - serviceConfigClassName); + .addAnnotation(Override.class) + .addModifiers(Modifier.PROTECTED, Modifier.FINAL) + .returns(clientInterfaceName) + .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", + SdkClientConfiguration.class) + .addStatement("this.validateClientOptions(clientConfiguration)") + .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" + + "(clientConfiguration)", + serviceConfigClassName); builder.addStatement("$1T client = new $2T(serviceClientConfiguration, clientConfiguration)", clientInterfaceName, clientClassName); @@ -156,6 +164,34 @@ private MethodSpec bearerTokenProviderMethod() { .build(); } + private MethodSpec multipartEnabledMethod(MultipartCustomization multipartCustomization) { + ClassName mulitpartConfigClassName = + PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); + return MethodSpec.methodBuilder("multipartEnabled") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addStatement("clientContextParams.put($T.MULTIPART_ENABLED_KEY, enabled)", + mulitpartConfigClassName) + .addStatement("return this") + .build(); + } + + private MethodSpec multipartConfigMethods(MultipartCustomization multipartCustomization) { + ClassName mulitpartConfigClassName = + PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); + return MethodSpec.methodBuilder("multipartConfiguration") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfig").build()) + .returns(builderInterfaceName) + .addStatement("clientContextParams.put($T.MULTIPART_CONFIGURATION_KEY, multipartConfig)", + mulitpartConfigClassName) + .addStatement("return this") + .build(); + } + private MethodSpec initializeServiceClientConfigMethod() { return MethodSpec.methodBuilder("initializeServiceClientConfig").addModifiers(Modifier.PRIVATE) .addParameter(SdkClientConfiguration.class, "clientConfig") diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java index 5348972b5df9..df62f97ae7c0 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java @@ -17,34 +17,97 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.CodeBlock; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; +import java.util.function.Consumer; +import javax.lang.model.element.Modifier; import software.amazon.awssdk.awscore.client.builder.AwsAsyncClientBuilder; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; public class AsyncClientBuilderInterface implements ClassSpec { + private static final Logger log = Logger.loggerFor(AsyncClientBuilderInterface.class); + private final ClassName builderInterfaceName; private final ClassName clientInterfaceName; private final ClassName baseBuilderInterfaceName; + private final IntermediateModel model; public AsyncClientBuilderInterface(IntermediateModel model) { String basePackage = model.getMetadata().getFullClientPackageName(); this.clientInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncInterface()); this.builderInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncBuilderInterface()); this.baseBuilderInterfaceName = ClassName.get(basePackage, model.getMetadata().getBaseBuilderInterface()); + this.model = model; } @Override public TypeSpec poetSpec() { - return PoetUtils.createInterfaceBuilder(builderInterfaceName) - .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), - builderInterfaceName, clientInterfaceName)) - .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, - builderInterfaceName, clientInterfaceName)) - .addJavadoc(getJavadoc()) - .build(); + TypeSpec.Builder builder = PoetUtils + .createInterfaceBuilder(builderInterfaceName) + .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), + builderInterfaceName, clientInterfaceName)) + .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, + builderInterfaceName, clientInterfaceName)) + .addJavadoc(getJavadoc()); + + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + includeMultipartMethod(builder, multipartCustomization); + } + return builder.build(); + } + + private void includeMultipartMethod(TypeSpec.Builder builder, MultipartCustomization multipartCustomization) { + log.debug(() -> String.format("Adding multipart config methods to builder interface for service '%s'", + model.getMetadata().getServiceId())); + + // .multipartEnabled(Boolean) + builder.addMethod( + MethodSpec.methodBuilder("multipartEnabled") + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartEnableMethodDoc())) + .build()); + + // .multipartConfiguration(MultipartConfiguration) + String multiPartConfigMethodName = "multipartConfiguration"; + String multipartConfigClass = Validate.notNull(multipartCustomization.getMultipartConfigurationClass(), + "'multipartConfigurationClass' must be defined"); + ClassName mulitpartConfigClassName = PoetUtils.classNameFromFqcn(multipartConfigClass); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfiguration").build()) + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); + + // .multipartConfiguration(Consumer) + ClassName mulitpartConfigBuilderClassName = PoetUtils.classNameFromFqcn(multipartConfigClass + ".Builder"); + ParameterizedTypeName consumerBuilderType = ParameterizedTypeName.get(ClassName.get(Consumer.class), + mulitpartConfigBuilderClassName); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(consumerBuilderType, "multipartConfiguration").build()) + .addStatement("$T builder = $T.builder()", + mulitpartConfigBuilderClassName, + mulitpartConfigClassName) + .addStatement("multipartConfiguration.accept(builder)") + .addStatement("return multipartConfiguration(builder.build())") + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); } @Override diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java index 63dcf2ddc88f..03cf42afe5df 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java @@ -117,7 +117,7 @@ protected static void deleteBucketAndAllContents(String bucketName) { S3TestUtils.deleteBucketAndAllContents(s3, bucketName); } - private static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { + protected static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { private final String clientName; private final ClientType clientType; diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java index 6db434526fb9..f9f65a33ad17 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; @@ -58,6 +59,7 @@ public class S3ClientMultiPartCopyIntegrationTest extends S3IntegrationTestBase private static final long SMALL_OBJ_SIZE = 1024 * 1024; private static S3AsyncClient s3CrtAsyncClient; private static S3AsyncClient s3MpuClient; + @BeforeAll public static void setUp() throws Exception { S3IntegrationTestBase.setUp(); @@ -66,7 +68,13 @@ public static void setUp() throws Exception { .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) .build(); - s3MpuClient = new MultipartS3AsyncClient(s3Async); + s3MpuClient = S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -158,7 +166,7 @@ private static byte[] generateSecretKey() { private void createOriginalObject(byte[] originalContent, String originalKey) { s3CrtAsyncClient.putObject(r -> r.bucket(BUCKET) - .key(originalKey), + .key(originalKey), AsyncRequestBody.fromBytes(originalContent)).join(); } diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java index cb72906943b9..77dbac9c570b 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.reactivestreams.Subscriber; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.internal.async.FileAsyncRequestBody; @@ -66,7 +67,14 @@ public static void setup() throws Exception { testFile = File.createTempFile("SplittingPublisherTest", UUID.randomUUID().toString()); Files.write(testFile.toPath(), CONTENT); - mpuS3Client = new MultipartS3AsyncClient(s3Async); + mpuS3Client = S3AsyncClient + .builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -81,8 +89,9 @@ void putObject_fileRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); mpuS3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(testFile.length()); byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); @@ -95,8 +104,9 @@ void putObject_byteAsyncRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromBytes(bytes); mpuS3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(OBJ_SIZE); byte[] expectedSum = ChecksumUtils.computeCheckSum(new ByteArrayInputStream(bytes)); @@ -120,8 +130,9 @@ public void subscribe(Subscriber s) { } }).get(30, SECONDS); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(testFile.length()); byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java index 2dbb61091da2..89c6982bb83e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java @@ -15,6 +15,9 @@ package software.amazon.awssdk.services.s3.internal.client; +import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_CONFIGURATION_KEY; +import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_ENABLED_KEY; + import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -23,6 +26,8 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.endpoints.S3ClientContextParams; import software.amazon.awssdk.services.s3.internal.crossregion.S3CrossRegionAsyncClient; +import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.ConditionalDecorator; @@ -36,14 +41,26 @@ public S3AsyncClient decorate(S3AsyncClient base, SdkClientConfiguration clientConfiguration, AttributeMap clientContextParams) { List> decorators = new ArrayList<>(); - decorators.add(ConditionalDecorator.create(isCrossRegionEnabledAsync(clientContextParams), - S3CrossRegionAsyncClient::new)); + decorators.add(ConditionalDecorator.create( + isCrossRegionEnabledAsync(clientContextParams), + S3CrossRegionAsyncClient::new)); + + decorators.add(ConditionalDecorator.create( + isMultipartEnable(clientContextParams), + client -> { + MultipartConfiguration multipartConfiguration = clientContextParams.get(MULTIPART_CONFIGURATION_KEY); + return new MultipartS3AsyncClient(client, multipartConfiguration); + })); return ConditionalDecorator.decorate(base, decorators); } private Predicate isCrossRegionEnabledAsync(AttributeMap clientContextParams) { Boolean crossRegionEnabled = clientContextParams.get(S3ClientContextParams.CROSS_REGION_ACCESS_ENABLED); - return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); + return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); } + private Predicate isMultipartEnable(AttributeMap clientContextParams) { + Boolean multipartEnabled = clientContextParams.get(MULTIPART_ENABLED_KEY); + return client -> multipartEnabled != null && multipartEnabled.booleanValue(); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java index 31b947bb89c5..c776c449f555 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java @@ -16,6 +16,8 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -23,6 +25,7 @@ import java.util.stream.IntStream; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.internal.crt.UploadPartCopyRequestIterable; import software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper; import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; @@ -67,8 +70,11 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb CompletableFuture returnFuture = new CompletableFuture<>(); try { + CopyObjectRequest copyObjectRequestWithUserAgent = + UserAgentUtils.applyUserAgentInfo(copyObjectRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture headFuture = - s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest)); + s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequestWithUserAgent)); // Ensure cancellations are forwarded to the head future CompletableFutureUtils.forwardExceptionTo(returnFuture, headFuture); @@ -78,7 +84,7 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb genericMultipartHelper.handleException(returnFuture, () -> "Failed to retrieve metadata from the source " + "object", throwable); } else { - doCopyObject(copyObjectRequest, returnFuture, headObjectResponse); + doCopyObject(copyObjectRequestWithUserAgent, returnFuture, headObjectResponse); } }); } catch (Throwable throwable) { @@ -105,7 +111,9 @@ private void copyInParts(CopyObjectRequest copyObjectRequest, Long contentLength, CompletableFuture returnFuture) { - CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest); + CreateMultipartUploadRequest request = UserAgentUtils + .applyUserAgentInfo(SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest), + c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -170,7 +178,8 @@ private CompletableFuture completeMultipartUplo .parts(parts) .build()) .build(); - + completeMultipartUploadRequest = UserAgentUtils.applyUserAgentInfo(completeMultipartUploadRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -201,7 +210,11 @@ private void sendIndividualUploadPartCopy(String uploadId, log.debug(() -> "Sending uploadPartCopyRequest with range: " + uploadPartCopyRequest.copySourceRange() + " uploadId: " + uploadId); - CompletableFuture uploadPartCopyFuture = s3AsyncClient.uploadPartCopy(uploadPartCopyRequest); + UploadPartCopyRequest uploadPartCopyRequestWithUserAgent = + UserAgentUtils.applyUserAgentInfo(uploadPartCopyRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); + CompletableFuture uploadPartCopyFuture = + s3AsyncClient.uploadPartCopy(uploadPartCopyRequestWithUserAgent); CompletableFuture convertFuture = uploadPartCopyFuture.thenApply(uploadPartCopyResponse -> @@ -225,8 +238,11 @@ private static CompletedPart convertUploadPartCopyResponse(AtomicReferenceArray< private void copyInOneChunk(CopyObjectRequest copyObjectRequest, CompletableFuture returnFuture) { + CopyObjectRequest copyObjectRequestWithUserAgent = + UserAgentUtils.applyUserAgentInfo(copyObjectRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture copyObjectFuture = - s3AsyncClient.copyObject(copyObjectRequest); + s3AsyncClient.copyObject(copyObjectRequestWithUserAgent); CompletableFutureUtils.forwardExceptionTo(returnFuture, copyObjectFuture); CompletableFutureUtils.forwardResultTo(copyObjectFuture, returnFuture); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java index 905c1bc928ea..b046602da139 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; + import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -25,6 +27,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; @@ -91,7 +94,8 @@ public CompletableFuture completeMultipartUploa .parts(parts) .build()) .build(); - + completeMultipartUploadRequest = UserAgentUtils.applyUserAgentInfo(completeMultipartUploadRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -125,7 +129,10 @@ public BiFunction handleExcept public void cleanUpParts(String uploadId, AbortMultipartUploadRequest.Builder abortMultipartUploadRequest) { log.debug(() -> "Aborting multipart upload: " + uploadId); - s3AsyncClient.abortMultipartUpload(abortMultipartUploadRequest.uploadId(uploadId).build()) + AbortMultipartUploadRequest request = UserAgentUtils + .applyUserAgentInfo(abortMultipartUploadRequest.uploadId(uploadId).build(), + c -> c.addApiName(USER_AGENT_API_NAME)); + s3AsyncClient.abortMultipartUpload(request) .exceptionally(throwable -> { log.warn(() -> String.format("Failed to abort previous multipart upload " + "(id: %s)" diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index a4b3147254f9..8acd989dccca 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -18,6 +18,7 @@ import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -25,11 +26,17 @@ import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.utils.Validate; // This is just a temporary class for testing //TODO: change this @SdkInternalApi public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { + + public static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); + + private static final long DEFAULT_MIN_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024; @@ -37,11 +44,23 @@ public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private final UploadObjectHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; - public MultipartS3AsyncClient(S3AsyncClient delegate) { + public MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) { super(delegate); - // TODO: pass a config object to the upload helper instead - mpuHelper = new UploadObjectHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD, DEFAULT_MAX_MEMORY); - copyObjectHelper = new CopyObjectHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD); + MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration, + MultipartConfiguration.builder()::build); + long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(), + () -> DEFAULT_MIN_PART_SIZE_IN_BYTES); + long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(), + () -> DEFAULT_THRESHOLD); + long maximumMemoryUsageInBytes = Validate.getOrDefault(validConfiguration.maximumMemoryUsageInBytes(), + () -> computeMaxMemoryUsage(validConfiguration)); + mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, maximumMemoryUsageInBytes); + copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); + } + + private long computeMaxMemoryUsage(MultipartConfiguration multipartConfiguration) { + return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 2 + : DEFAULT_MAX_MEMORY; } @Override diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index 1228e577fcd1..34c0dce23867 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; import static software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils.toAbortMultipartUploadRequest; import java.util.Collection; @@ -24,6 +25,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -36,8 +38,8 @@ import software.amazon.awssdk.utils.Pair; /** - * A base class contains common logic used by {@link UploadWithUnknownContentLengthHelper} - * and {@link UploadWithKnownContentLengthHelper}. + * A base class contains common logic used by {@link UploadWithUnknownContentLengthHelper} and + * {@link UploadWithKnownContentLengthHelper}. */ @SdkInternalApi public final class MultipartUploadHelper { @@ -66,6 +68,8 @@ public MultipartUploadHelper(S3AsyncClient s3AsyncClient, CompletableFuture createMultipartUpload(PutObjectRequest putObjectRequest, CompletableFuture returnFuture) { CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); + request = UserAgentUtils.applyUserAgentInfo(request, + c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -94,7 +98,8 @@ CompletableFuture sendIndividualUploadPartRequest(String uploadId Consumer completedPartsConsumer, Collection> futures, Pair requestPair) { - UploadPartRequest uploadPartRequest = requestPair.left(); + UploadPartRequest uploadPartRequest = UserAgentUtils.applyUserAgentInfo(requestPair.left(), + c -> c.addApiName(USER_AGENT_API_NAME)); Integer partNumber = uploadPartRequest.partNumber(); log.debug(() -> "Sending uploadPartRequest: " + uploadPartRequest.partNumber() + " uploadId: " + uploadId + " " + "contentLength " + requestPair.right().contentLength()); @@ -139,6 +144,8 @@ static CompletedPart convertUploadPartResponse(Consumer consumer, void uploadInOneChunk(PutObjectRequest putObjectRequest, AsyncRequestBody asyncRequestBody, CompletableFuture returnFuture) { + putObjectRequest = UserAgentUtils.applyUserAgentInfo(putObjectRequest, + c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture putObjectResponseCompletableFuture = s3AsyncClient.putObject(putObjectRequest, asyncRequestBody); CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectResponseCompletableFuture); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java new file mode 100644 index 000000000000..64e85858be91 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -0,0 +1,206 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.multipart; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Class that hold configuration properties related to multipart operation for a {@link S3AsyncClient}. Passing this class to the + * {@link S3AsyncClientBuilder#multipartConfiguration(MultipartConfiguration)} will enable automatic conversion of + * {@link S3AsyncClient#putObject(Consumer, AsyncRequestBody)}, {@link S3AsyncClient#copyObject(CopyObjectRequest)} to their + * respective multipart operation. + *

+ * Note: The multipart operation for {@link S3AsyncClient#getObject(GetObjectRequest, AsyncResponseTransformer)} is + * temporarily disabled and will result in throwing a {@link UnsupportedOperationException} if called when configured for + * multipart operation. + */ +@SdkPublicApi +public final class MultipartConfiguration implements ToCopyableBuilder { + public static final AttributeMap.Key MULTIPART_CONFIGURATION_KEY = + new AttributeMap.Key(MultipartConfiguration.class){}; + public static final AttributeMap.Key MULTIPART_ENABLED_KEY = + new AttributeMap.Key(Boolean.class){}; + + private final Long thresholdInBytes; + private final Long minimumPartSizeInBytes; + private final Long maximumMemoryUsageInBytes; + + private MultipartConfiguration(DefaultMultipartConfigBuilder builder) { + this.thresholdInBytes = builder.thresholdInBytes; + this.minimumPartSizeInBytes = builder.minimumPartSizeInBytes; + this.maximumMemoryUsageInBytes = builder.maximumMemoryUsageInBytes; + } + + public static Builder builder() { + return new DefaultMultipartConfigBuilder(); + } + + @Override + public Builder toBuilder() { + return builder() + .maximumMemoryUsageInBytes(maximumMemoryUsageInBytes) + .minimumPartSizeInBytes(minimumPartSizeInBytes) + .thresholdInBytes(thresholdInBytes); + } + + /** + * Indicates the value of the configured threshold, in bytes. Any request whose size is less than the configured value will + * not + * use multipart operation + * @return the value of the configured threshold. + */ + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + /** + * Indicated the size, in bytes, of each individual part of the part requests. The actual part size used might be bigger to + * conforms to + * the maximum + * number of parts allowed per multipart requests. + * @return the value of the configured part size. + */ + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + /** + * The maximum memory, in bytes, that the SDK will use to buffer requests content into memory. + * @return the value of the configured maximum memory usage. + */ + public Long maximumMemoryUsageInBytes() { + return this.maximumMemoryUsageInBytes; + } + + /** + * Builder for a {@link MultipartConfiguration}. + */ + public interface Builder extends CopyableBuilder { + + /** + * Configures the minimum number of bytes of the body of the request required for requests to be converted to their + * multipart equivalent. Only taken into account when converting {@code putObject} and {@code copyObject} requests. + * Any request whose size is less than the configured value will not use multipart operation, + * even if multipart is enabled via {@link S3AsyncClientBuilder#multipartEnabled(Boolean)}. + *

+ * + * Default value: 8 Mib + * + * @param thresholdInBytes the value of the threshold to set. + * @return an instance of this builder. + */ + Builder thresholdInBytes(Long thresholdInBytes); + + /** + * Indicates the value of the configured threshold. + * @return the value of the threshold. + */ + Long thresholdInBytes(); + + /** + * Configures the part size, in bytes, to be used in each individual part requests. + *

+ * When uploading large payload, the size of the payload of each individual part requests might actually be + * bigger than + * the configured value since there is a limit to the maximum number of parts possible per multipart request. If the + * configured part size would lead to a number of parts higher than the maximum allowed, a larger part size will be + * calculated instead to allow fewer part to be uploaded, to avoid the limit imposed on the maximum number of parts. + *

+ * In the case where the {@code minimumPartSizeInBytes} is set to a value higher than the {@code thresholdInBytes}, when + * the client receive a request with a size smaller than a single part multipart operation will NOT be performed + * even if the size of the request is larger than the threshold. + *

+ * Default value: 8 Mib + * + * @param minimumPartSizeInBytes the value of the part size to set + * @return an instance of this builder. + */ + Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes); + + /** + * Indicated the value of the part configured size. + * @return the value of the part size + */ + Long minimumPartSizeInBytes(); + + /** + * Configures the maximum amount of memory, in bytes, the SDK will use to buffer content of requests in memory. + * Increasing this value my lead to better performance at the cost of using more memory. + *

+ * Default value: If not specified, the SDK will use the equivalent of two parts worth of memory, so 16 Mib by default. + * + * @param maximumMemoryUsageInBytes the value of the maximum memory usage. + * @return an instance of this builder. + */ + Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes); + + /** + * Indicates the value of the maximum memory usage that the SDK will use. + * @return the value of the maximum memory usage. + */ + Long maximumMemoryUsageInBytes(); + } + + private static class DefaultMultipartConfigBuilder implements Builder { + private Long thresholdInBytes; + private Long minimumPartSizeInBytes; + private Long maximumMemoryUsageInBytes; + + public Builder thresholdInBytes(Long thresholdInBytes) { + this.thresholdInBytes = thresholdInBytes; + return this; + } + + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + public Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes) { + this.minimumPartSizeInBytes = minimumPartSizeInBytes; + return this; + } + + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + @Override + public Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes) { + this.maximumMemoryUsageInBytes = maximumMemoryUsageInBytes; + return this; + } + + @Override + public Long maximumMemoryUsageInBytes() { + return maximumMemoryUsageInBytes; + } + + @Override + public MultipartConfiguration build() { + return new MultipartConfiguration(this); + } + } +} diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 1a1efb76c5f4..053ef58171a6 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -236,6 +236,11 @@ "syncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3SyncClientDecorator", "asyncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3AsyncClientDecorator", "useGlobalEndpoint": true, + "multipartCustomization": { + "multipartConfigurationClass": "software.amazon.awssdk.services.s3.multipart.MultipartConfiguration", + "multipartConfigMethodDoc": "Configuration for multipart operation of this client.", + "multipartEnableMethodDoc": "Enables automatic conversion of put and copy method to their equivalent multipart operation." + }, "interceptors": [ "software.amazon.awssdk.services.s3.internal.handlers.PutObjectInterceptor", "software.amazon.awssdk.services.s3.internal.handlers.CreateBucketInterceptor", diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java new file mode 100644 index 000000000000..1bbd10213b7c --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; + +public class MultipartClientUserAgentTest { + private MockAsyncHttpClient mockAsyncHttpClient; + private UserAgentInterceptor userAgentInterceptor; + private S3AsyncClient s3Client; + + @BeforeEach + void init() { + this.mockAsyncHttpClient = new MockAsyncHttpClient(); + this.userAgentInterceptor = new UserAgentInterceptor(); + s3Client = S3AsyncClient.builder() + .httpClient(mockAsyncHttpClient) + .endpointOverride(URI.create("http://localhost")) + .overrideConfiguration(c -> c.addExecutionInterceptor(userAgentInterceptor)) + .multipartEnabled(true) + .multipartConfiguration(c -> c.minimumPartSizeInBytes(1024L).thresholdInBytes(1024L)) + .region(Region.US_EAST_1) + .build(); + } + + @Test + void validateUserAgent_put_oneChunk() throws Exception { + HttpExecuteResponse response = HttpExecuteResponse.builder() + .response(SdkHttpResponse.builder().statusCode(200).build()) + .build(); + mockAsyncHttpClient.stubResponses(response); + + s3Client.putObject(req -> req.key("key").bucket("bucket"), AsyncRequestBody.fromString("12345678")).get(); + + assertThat(userAgentInterceptor.apiNames).isNotNull(); + assertThat(userAgentInterceptor.apiNames) + .anyMatch(api -> "hll".equals(api.name()) && "s3Multipart".equals(api.version())); + } + + private static final class UserAgentInterceptor implements ExecutionInterceptor { + private final List apiNames = new ArrayList<>(); + + @Override + public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes executionAttributes) { + context.request().overrideConfiguration().ifPresent(c -> apiNames.addAll(c.apiNames())); + } + } + +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java new file mode 100644 index 000000000000..7a7366ce38a7 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; + +class S3MultipartClientBuilderTest { + + @Test + void multipartEnabledWithConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .multipartConfiguration(MultipartConfiguration.builder().build()) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartEnabledWithoutConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartDisabledWithConfig_shouldNotBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(false) + .multipartConfiguration(b -> b.maximumMemoryUsageInBytes(1024L)) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void noMultipart_shouldNotBeMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } +} From 19efa8b82a7bff0b49e9b50e40cc25ac76e2caa5 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Jul 2023 17:00:08 -0400 Subject: [PATCH 02/10] getObject(...) throw UnsupportedOperationException --- .../s3/internal/multipart/MultipartS3AsyncClient.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 8acd989dccca..c686dbbe1c60 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -20,10 +20,13 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; @@ -73,6 +76,13 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb return copyObjectHelper.copyObject(copyObjectRequest); } + @Override + public CompletableFuture getObject( + GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { + throw new UnsupportedOperationException( + "Multipart download is not yet supported. Instead use the CRT based S3 client for multipart download."); + } + @Override public void close() { delegate().close(); From 26672193c04f5d32f5378f266f71e3902b6e403c Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 28 Jul 2023 15:04:15 -0400 Subject: [PATCH 03/10] Use user agent for all requests in MultipartS3Client --- .../internal/multipart/CopyObjectHelper.java | 28 ++++--------------- .../multipart/GenericMultipartHelper.java | 9 +----- .../multipart/MultipartS3AsyncClient.java | 10 +++++++ .../multipart/MultipartUploadHelper.java | 9 +----- .../MultipartClientUserAgentTest.java | 3 +- 5 files changed, 18 insertions(+), 41 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java index c776c449f555..01c60b6f9a98 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java @@ -16,8 +16,6 @@ package software.amazon.awssdk.services.s3.internal.multipart; -import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -25,10 +23,7 @@ import java.util.stream.IntStream; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.internal.crt.UploadPartCopyRequestIterable; -import software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper; -import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; @@ -70,11 +65,8 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb CompletableFuture returnFuture = new CompletableFuture<>(); try { - CopyObjectRequest copyObjectRequestWithUserAgent = - UserAgentUtils.applyUserAgentInfo(copyObjectRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture headFuture = - s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequestWithUserAgent)); + s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest)); // Ensure cancellations are forwarded to the head future CompletableFutureUtils.forwardExceptionTo(returnFuture, headFuture); @@ -84,7 +76,7 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb genericMultipartHelper.handleException(returnFuture, () -> "Failed to retrieve metadata from the source " + "object", throwable); } else { - doCopyObject(copyObjectRequestWithUserAgent, returnFuture, headObjectResponse); + doCopyObject(copyObjectRequest, returnFuture, headObjectResponse); } }); } catch (Throwable throwable) { @@ -111,9 +103,7 @@ private void copyInParts(CopyObjectRequest copyObjectRequest, Long contentLength, CompletableFuture returnFuture) { - CreateMultipartUploadRequest request = UserAgentUtils - .applyUserAgentInfo(SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest), - c -> c.addApiName(USER_AGENT_API_NAME)); + CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -178,8 +168,6 @@ private CompletableFuture completeMultipartUplo .parts(parts) .build()) .build(); - completeMultipartUploadRequest = UserAgentUtils.applyUserAgentInfo(completeMultipartUploadRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -210,11 +198,8 @@ private void sendIndividualUploadPartCopy(String uploadId, log.debug(() -> "Sending uploadPartCopyRequest with range: " + uploadPartCopyRequest.copySourceRange() + " uploadId: " + uploadId); - UploadPartCopyRequest uploadPartCopyRequestWithUserAgent = - UserAgentUtils.applyUserAgentInfo(uploadPartCopyRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture uploadPartCopyFuture = - s3AsyncClient.uploadPartCopy(uploadPartCopyRequestWithUserAgent); + s3AsyncClient.uploadPartCopy(uploadPartCopyRequest); CompletableFuture convertFuture = uploadPartCopyFuture.thenApply(uploadPartCopyResponse -> @@ -238,11 +223,8 @@ private static CompletedPart convertUploadPartCopyResponse(AtomicReferenceArray< private void copyInOneChunk(CopyObjectRequest copyObjectRequest, CompletableFuture returnFuture) { - CopyObjectRequest copyObjectRequestWithUserAgent = - UserAgentUtils.applyUserAgentInfo(copyObjectRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture copyObjectFuture = - s3AsyncClient.copyObject(copyObjectRequestWithUserAgent); + s3AsyncClient.copyObject(copyObjectRequest); CompletableFutureUtils.forwardExceptionTo(returnFuture, copyObjectFuture); CompletableFutureUtils.forwardResultTo(copyObjectFuture, returnFuture); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java index b046602da139..38e76394958e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java @@ -15,8 +15,6 @@ package software.amazon.awssdk.services.s3.internal.multipart; -import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; - import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -27,7 +25,6 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; @@ -94,8 +91,6 @@ public CompletableFuture completeMultipartUploa .parts(parts) .build()) .build(); - completeMultipartUploadRequest = UserAgentUtils.applyUserAgentInfo(completeMultipartUploadRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -129,9 +124,7 @@ public BiFunction handleExcept public void cleanUpParts(String uploadId, AbortMultipartUploadRequest.Builder abortMultipartUploadRequest) { log.debug(() -> "Aborting multipart upload: " + uploadId); - AbortMultipartUploadRequest request = UserAgentUtils - .applyUserAgentInfo(abortMultipartUploadRequest.uploadId(uploadId).build(), - c -> c.addApiName(USER_AGENT_API_NAME)); + AbortMultipartUploadRequest request = abortMultipartUploadRequest.uploadId(uploadId).build(); s3AsyncClient.abortMultipartUpload(request) .exceptionally(throwable -> { log.warn(() -> String.format("Failed to abort previous multipart upload " diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index c686dbbe1c60..8d823ea95890 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -17,18 +17,21 @@ import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.utils.Validate; @@ -66,6 +69,13 @@ private long computeMaxMemoryUsage(MultipartConfiguration multipartConfiguration : DEFAULT_MAX_MEMORY; } + @Override + protected CompletableFuture invokeOperation(T request, Function> operation) { + T requestWithUserAgent = UserAgentUtils.applyUserAgentInfo(request, c -> c.addApiName(USER_AGENT_API_NAME)); + return operation.apply(requestWithUserAgent); + } + @Override public CompletableFuture putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) { return mpuHelper.uploadObject(putObjectRequest, requestBody); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index 34c0dce23867..9754d284f5b9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -16,7 +16,6 @@ package software.amazon.awssdk.services.s3.internal.multipart; -import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; import static software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils.toAbortMultipartUploadRequest; import java.util.Collection; @@ -25,7 +24,6 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -68,8 +66,6 @@ public MultipartUploadHelper(S3AsyncClient s3AsyncClient, CompletableFuture createMultipartUpload(PutObjectRequest putObjectRequest, CompletableFuture returnFuture) { CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); - request = UserAgentUtils.applyUserAgentInfo(request, - c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -98,8 +94,7 @@ CompletableFuture sendIndividualUploadPartRequest(String uploadId Consumer completedPartsConsumer, Collection> futures, Pair requestPair) { - UploadPartRequest uploadPartRequest = UserAgentUtils.applyUserAgentInfo(requestPair.left(), - c -> c.addApiName(USER_AGENT_API_NAME)); + UploadPartRequest uploadPartRequest = requestPair.left(); Integer partNumber = uploadPartRequest.partNumber(); log.debug(() -> "Sending uploadPartRequest: " + uploadPartRequest.partNumber() + " uploadId: " + uploadId + " " + "contentLength " + requestPair.right().contentLength()); @@ -144,8 +139,6 @@ static CompletedPart convertUploadPartResponse(Consumer consumer, void uploadInOneChunk(PutObjectRequest putObjectRequest, AsyncRequestBody asyncRequestBody, CompletableFuture returnFuture) { - putObjectRequest = UserAgentUtils.applyUserAgentInfo(putObjectRequest, - c -> c.addApiName(USER_AGENT_API_NAME)); CompletableFuture putObjectResponseCompletableFuture = s3AsyncClient.putObject(putObjectRequest, asyncRequestBody); CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectResponseCompletableFuture); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java index 1bbd10213b7c..d9e5a809b2fa 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ApiName; -import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.interceptor.Context; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; @@ -59,7 +58,7 @@ void validateUserAgent_put_oneChunk() throws Exception { .build(); mockAsyncHttpClient.stubResponses(response); - s3Client.putObject(req -> req.key("key").bucket("bucket"), AsyncRequestBody.fromString("12345678")).get(); + s3Client.headObject(req -> req.key("mock").bucket("mock")).get(); assertThat(userAgentInterceptor.apiNames).isNotNull(); assertThat(userAgentInterceptor.apiNames) From bcbbf98419488641d0a40ce4a661aac3f48290a4 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 28 Jul 2023 15:09:39 -0400 Subject: [PATCH 04/10] MultipartS3AsyncClient javadoc + API_NAME private --- .../s3/internal/multipart/MultipartS3AsyncClient.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 8d823ea95890..7168747818d9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -35,12 +35,15 @@ import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.utils.Validate; -// This is just a temporary class for testing -//TODO: change this +/** + * An {@link S3AsyncClient} that automatically converts put, copy requests to their respective multipart call. + * Note: get is not yet supported. + * @see MultipartConfiguration + */ @SdkInternalApi public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { - public static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); + private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); private static final long DEFAULT_MIN_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; From 7e82f380ce3e3518690c51427eb7fe62a1558be1 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 28 Jul 2023 15:34:15 -0400 Subject: [PATCH 05/10] use `maximumMemoryUsageInBytes` --- .../internal/multipart/CopyObjectHelper.java | 4 +++ .../multipart/MultipartS3AsyncClient.java | 17 ++++++----- .../UploadWithKnownContentLengthHelper.java | 4 +++ .../s3/multipart/MultipartConfiguration.java | 28 +++++++++---------- .../S3MultipartClientBuilderTest.java | 2 +- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java index 01c60b6f9a98..16294ff8f065 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java @@ -128,6 +128,10 @@ private void doCopyInParts(CopyObjectRequest copyObjectRequest, long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); + if (optimalPartSize > partSizeInBytes) { + log.debug(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } log.debug(() -> String.format("Starting multipart copy with partCount: %s, optimalPartSize: %s", partCount, optimalPartSize)); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 7168747818d9..666b92e91273 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -45,11 +45,10 @@ public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); - private static final long DEFAULT_MIN_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; - private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; + private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024; private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024; + private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 2; - private static final long DEFAULT_MAX_MEMORY = DEFAULT_PART_SIZE_IN_BYTES * 2; private final UploadObjectHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; @@ -58,18 +57,18 @@ public MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mul MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration, MultipartConfiguration.builder()::build); long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(), - () -> DEFAULT_MIN_PART_SIZE_IN_BYTES); + () -> DEFAULT_MIN_PART_SIZE); long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(), () -> DEFAULT_THRESHOLD); - long maximumMemoryUsageInBytes = Validate.getOrDefault(validConfiguration.maximumMemoryUsageInBytes(), - () -> computeMaxMemoryUsage(validConfiguration)); - mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, maximumMemoryUsageInBytes); + long apiCallBufferSizeInBytes = Validate.getOrDefault(validConfiguration.apiCallBufferSizeInBytes(), + () -> computeApiCallBufferSize(validConfiguration)); + mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, apiCallBufferSizeInBytes); copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); } - private long computeMaxMemoryUsage(MultipartConfiguration multipartConfiguration) { + private long computeApiCallBufferSize(MultipartConfiguration multipartConfiguration) { return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 2 - : DEFAULT_MAX_MEMORY; + : DEFAULT_API_CALL_BUFFER_SIZE; } @Override diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java index e8bef01ab81b..a00b9eb9189d 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java @@ -112,6 +112,10 @@ private void doUploadInParts(Pair request, long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); + if (optimalPartSize > partSizeInBytes) { + log.debug(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } log.debug(() -> String.format("Starting multipart upload with partCount: %d, optimalPartSize: %d", partCount, optimalPartSize)); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java index 64e85858be91..8f0307555f1b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -46,12 +46,12 @@ public final class MultipartConfiguration implements ToCopyableBuilder

* Default value: If not specified, the SDK will use the equivalent of two parts worth of memory, so 16 Mib by default. * - * @param maximumMemoryUsageInBytes the value of the maximum memory usage. + * @param apiCallBufferSizeInBytes the value of the maximum memory usage. * @return an instance of this builder. */ - Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes); + Builder apiCallBufferSizeInBytes(Long apiCallBufferSizeInBytes); /** * Indicates the value of the maximum memory usage that the SDK will use. * @return the value of the maximum memory usage. */ - Long maximumMemoryUsageInBytes(); + Long apiCallBufferSizeInBytes(); } private static class DefaultMultipartConfigBuilder implements Builder { private Long thresholdInBytes; private Long minimumPartSizeInBytes; - private Long maximumMemoryUsageInBytes; + private Long apiCallBufferSizeInBytes; public Builder thresholdInBytes(Long thresholdInBytes) { this.thresholdInBytes = thresholdInBytes; @@ -188,14 +188,14 @@ public Long minimumPartSizeInBytes() { } @Override - public Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes) { - this.maximumMemoryUsageInBytes = maximumMemoryUsageInBytes; + public Builder apiCallBufferSizeInBytes(Long maximumMemoryUsageInBytes) { + this.apiCallBufferSizeInBytes = maximumMemoryUsageInBytes; return this; } @Override - public Long maximumMemoryUsageInBytes() { - return maximumMemoryUsageInBytes; + public Long apiCallBufferSizeInBytes() { + return apiCallBufferSizeInBytes; } @Override diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java index 7a7366ce38a7..510d441c4caa 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java @@ -47,7 +47,7 @@ void multipartEnabledWithoutConfig_shouldBuildMultipartClient() { void multipartDisabledWithConfig_shouldNotBuildMultipartClient() { S3AsyncClient client = S3AsyncClient.builder() .multipartEnabled(false) - .multipartConfiguration(b -> b.maximumMemoryUsageInBytes(1024L)) + .multipartConfiguration(b -> b.apiCallBufferSizeInBytes(1024L)) .region(Region.US_EAST_1) .build(); assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); From f197c2eed4ee5b29f5b6f8921ab334a278e8b559 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 28 Jul 2023 17:16:20 -0400 Subject: [PATCH 06/10] fix problem with UserAgent, cleanup --- .../config/customization/ServiceConfig.java | 16 ---------- .../S3ClientMultiPartCopyIntegrationTest.java | 2 -- ...ltipartClientPutObjectIntegrationTest.java | 6 ---- .../client/S3AsyncClientDecorator.java | 2 +- .../multipart/MultipartS3AsyncClient.java | 30 +++++++++++-------- .../MultipartClientUserAgentTest.java | 13 +++++--- 6 files changed, 28 insertions(+), 41 deletions(-) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java index 3dc4b6291b5e..b0c0db862df2 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java @@ -15,8 +15,6 @@ package software.amazon.awssdk.codegen.model.config.customization; -import software.amazon.awssdk.utils.ToString; - public class ServiceConfig { /** * Specifies the name of the client configuration class to use if a service @@ -114,18 +112,4 @@ public boolean hasAccelerateModeEnabledProperty() { public void setHasAccelerateModeEnabledProperty(boolean hasAccelerateModeEnabledProperty) { this.hasAccelerateModeEnabledProperty = hasAccelerateModeEnabledProperty; } - - @Override - public String toString() { - return ToString.builder("ServiceConfig") - .add("className", className) - .add("hasDualstackProperty", hasDualstackProperty) - .add("hasFipsProperty", hasFipsProperty) - .add("hasUseArnRegionProperty", hasUseArnRegionProperty) - .add("hasMultiRegionEnabledProperty", hasMultiRegionEnabledProperty) - .add("hasPathStyleAccessEnabledProperty", hasPathStyleAccessEnabledProperty) - .add("hasAccelerateModeEnabledProperty", hasAccelerateModeEnabledProperty) - .add("hasCrossRegionAccessEnabledProperty", hasCrossRegionAccessEnabledProperty) - .build(); - } } diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java index f9f65a33ad17..fc4f31b76b1a 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java @@ -31,7 +31,6 @@ import javax.crypto.KeyGenerator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -42,7 +41,6 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.MetadataDirective; diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java index 77dbac9c570b..fa31b5453e5e 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java @@ -15,7 +15,6 @@ package software.amazon.awssdk.services.s3.multipart; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; @@ -27,10 +26,7 @@ import java.nio.file.Files; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -43,10 +39,8 @@ import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.utils.ChecksumUtils; -import software.amazon.awssdk.testutils.RandomTempFile; @Timeout(value = 30, unit = SECONDS) public class S3MultipartClientPutObjectIntegrationTest extends S3IntegrationTestBase { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java index 89c6982bb83e..a36e9b1053d9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java @@ -49,7 +49,7 @@ public S3AsyncClient decorate(S3AsyncClient base, isMultipartEnable(clientContextParams), client -> { MultipartConfiguration multipartConfiguration = clientContextParams.get(MULTIPART_CONFIGURATION_KEY); - return new MultipartS3AsyncClient(client, multipartConfiguration); + return MultipartS3AsyncClient.create(client, multipartConfiguration); })); return ConditionalDecorator.decorate(base, decorators); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 666b92e91273..5e706b50cd03 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -36,12 +36,13 @@ import software.amazon.awssdk.utils.Validate; /** - * An {@link S3AsyncClient} that automatically converts put, copy requests to their respective multipart call. - * Note: get is not yet supported. + * An {@link S3AsyncClient} that automatically converts put, copy requests to their respective multipart call. Note: get is not + * yet supported. + * * @see MultipartConfiguration */ @SdkInternalApi -public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { +public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); @@ -52,7 +53,7 @@ public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private final UploadObjectHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; - public MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) { + private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) { super(delegate); MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration, MultipartConfiguration.builder()::build); @@ -61,7 +62,7 @@ public MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mul long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(), () -> DEFAULT_THRESHOLD); long apiCallBufferSizeInBytes = Validate.getOrDefault(validConfiguration.apiCallBufferSizeInBytes(), - () -> computeApiCallBufferSize(validConfiguration)); + () -> computeApiCallBufferSize(validConfiguration)); mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, apiCallBufferSizeInBytes); copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); } @@ -71,13 +72,6 @@ private long computeApiCallBufferSize(MultipartConfiguration multipartConfigurat : DEFAULT_API_CALL_BUFFER_SIZE; } - @Override - protected CompletableFuture invokeOperation(T request, Function> operation) { - T requestWithUserAgent = UserAgentUtils.applyUserAgentInfo(request, c -> c.addApiName(USER_AGENT_API_NAME)); - return operation.apply(requestWithUserAgent); - } - @Override public CompletableFuture putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) { return mpuHelper.uploadObject(putObjectRequest, requestBody); @@ -99,4 +93,16 @@ public CompletableFuture getObject( public void close() { delegate().close(); } + + public static MultipartS3AsyncClient create(S3AsyncClient client, MultipartConfiguration multipartConfiguration) { + S3AsyncClient clientWithUserAgent = new DelegatingS3AsyncClient(client) { + @Override + protected CompletableFuture invokeOperation(T request, Function> operation) { + T requestWithUserAgent = UserAgentUtils.applyUserAgentInfo(request, c -> c.addApiName(USER_AGENT_API_NAME)); + return operation.apply(requestWithUserAgent); + } + }; + return new MultipartS3AsyncClient(clientWithUserAgent, multipartConfiguration); + } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java index d9e5a809b2fa..0f41c7c78e74 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java @@ -20,6 +20,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ApiName; @@ -32,7 +33,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; -public class MultipartClientUserAgentTest { +class MultipartClientUserAgentTest { private MockAsyncHttpClient mockAsyncHttpClient; private UserAgentInterceptor userAgentInterceptor; private S3AsyncClient s3Client; @@ -45,14 +46,19 @@ void init() { .httpClient(mockAsyncHttpClient) .endpointOverride(URI.create("http://localhost")) .overrideConfiguration(c -> c.addExecutionInterceptor(userAgentInterceptor)) + .multipartConfiguration(c -> c.minimumPartSizeInBytes(512L).thresholdInBytes(512L)) .multipartEnabled(true) - .multipartConfiguration(c -> c.minimumPartSizeInBytes(1024L).thresholdInBytes(1024L)) .region(Region.US_EAST_1) .build(); } + @AfterEach + void reset() { + this.mockAsyncHttpClient.reset(); + } + @Test - void validateUserAgent_put_oneChunk() throws Exception { + void validateUserAgent_nonMultipartMethod() throws Exception { HttpExecuteResponse response = HttpExecuteResponse.builder() .response(SdkHttpResponse.builder().statusCode(200).build()) .build(); @@ -60,7 +66,6 @@ void validateUserAgent_put_oneChunk() throws Exception { s3Client.headObject(req -> req.key("mock").bucket("mock")).get(); - assertThat(userAgentInterceptor.apiNames).isNotNull(); assertThat(userAgentInterceptor.apiNames) .anyMatch(api -> "hll".equals(api.name()) && "s3Multipart".equals(api.version())); } From e9814baad42d971df03af8c996111abfb86a4d1a Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Jul 2023 15:41:08 -0400 Subject: [PATCH 07/10] move contextParam keys to S3AsyncClientDecorator --- .../customization/MultipartCustomization.java | 18 +++++++++++++++ .../poet/builder/AsyncClientBuilderClass.java | 22 +++++++++---------- .../client/S3AsyncClientDecorator.java | 7 +++--- .../s3/multipart/MultipartConfiguration.java | 7 +----- .../codegen-resources/customization.config | 4 +++- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java index 6f87cc50ce6c..94264a9e5ec6 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java @@ -19,6 +19,8 @@ public class MultipartCustomization { private String multipartConfigurationClass; private String multipartConfigMethodDoc; private String multipartEnableMethodDoc; + private String contextParamEnabledKey; + private String contextParamConfigKey; public String getMultipartConfigurationClass() { return multipartConfigurationClass; @@ -43,4 +45,20 @@ public String getMultipartEnableMethodDoc() { public void setMultipartEnableMethodDoc(String multipartEnableMethodDoc) { this.multipartEnableMethodDoc = multipartEnableMethodDoc; } + + public String getContextParamEnabledKey() { + return contextParamEnabledKey; + } + + public void setContextParamEnabledKey(String contextParamEnabledKey) { + this.contextParamEnabledKey = contextParamEnabledKey; + } + + public String getContextParamConfigKey() { + return contextParamConfigKey; + } + + public void setContextParamConfigKey(String contextParamConfigKey) { + this.contextParamConfigKey = contextParamConfigKey; + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java index efe81f2cb9a1..3ff2b99ec98e 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java @@ -165,17 +165,15 @@ private MethodSpec bearerTokenProviderMethod() { } private MethodSpec multipartEnabledMethod(MultipartCustomization multipartCustomization) { - ClassName mulitpartConfigClassName = - PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); return MethodSpec.methodBuilder("multipartEnabled") - .addAnnotation(Override.class) - .addModifiers(Modifier.PUBLIC) - .returns(builderInterfaceName) - .addParameter(Boolean.class, "enabled") - .addStatement("clientContextParams.put($T.MULTIPART_ENABLED_KEY, enabled)", - mulitpartConfigClassName) - .addStatement("return this") - .build(); + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addStatement("clientContextParams.put($N, enabled)", + multipartCustomization.getContextParamEnabledKey()) + .addStatement("return this") + .build(); } private MethodSpec multipartConfigMethods(MultipartCustomization multipartCustomization) { @@ -186,8 +184,8 @@ private MethodSpec multipartConfigMethods(MultipartCustomization multipartCustom .addModifiers(Modifier.PUBLIC) .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfig").build()) .returns(builderInterfaceName) - .addStatement("clientContextParams.put($T.MULTIPART_CONFIGURATION_KEY, multipartConfig)", - mulitpartConfigClassName) + .addStatement("clientContextParams.put($N, multipartConfig)", + multipartCustomization.getContextParamConfigKey()) .addStatement("return this") .build(); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java index a36e9b1053d9..b751cb29c1b0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java @@ -15,9 +15,6 @@ package software.amazon.awssdk.services.s3.internal.client; -import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_CONFIGURATION_KEY; -import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_ENABLED_KEY; - import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -33,6 +30,10 @@ @SdkInternalApi public class S3AsyncClientDecorator { + public static final AttributeMap.Key MULTIPART_CONFIGURATION_KEY = + new AttributeMap.Key(MultipartConfiguration.class){}; + public static final AttributeMap.Key MULTIPART_ENABLED_KEY = + new AttributeMap.Key(Boolean.class){}; public S3AsyncClientDecorator() { } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java index 8f0307555f1b..acddfd02e91b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -23,7 +23,6 @@ import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -32,17 +31,13 @@ * {@link S3AsyncClientBuilder#multipartConfiguration(MultipartConfiguration)} will enable automatic conversion of * {@link S3AsyncClient#putObject(Consumer, AsyncRequestBody)}, {@link S3AsyncClient#copyObject(CopyObjectRequest)} to their * respective multipart operation. - *

+ *

* Note: The multipart operation for {@link S3AsyncClient#getObject(GetObjectRequest, AsyncResponseTransformer)} is * temporarily disabled and will result in throwing a {@link UnsupportedOperationException} if called when configured for * multipart operation. */ @SdkPublicApi public final class MultipartConfiguration implements ToCopyableBuilder { - public static final AttributeMap.Key MULTIPART_CONFIGURATION_KEY = - new AttributeMap.Key(MultipartConfiguration.class){}; - public static final AttributeMap.Key MULTIPART_ENABLED_KEY = - new AttributeMap.Key(Boolean.class){}; private final Long thresholdInBytes; private final Long minimumPartSizeInBytes; diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 053ef58171a6..ccddba62880c 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -239,7 +239,9 @@ "multipartCustomization": { "multipartConfigurationClass": "software.amazon.awssdk.services.s3.multipart.MultipartConfiguration", "multipartConfigMethodDoc": "Configuration for multipart operation of this client.", - "multipartEnableMethodDoc": "Enables automatic conversion of put and copy method to their equivalent multipart operation." + "multipartEnableMethodDoc": "Enables automatic conversion of put and copy method to their equivalent multipart operation.", + "contextParamEnabledKey": "S3AsyncClientDecorator.MULTIPART_ENABLED_KEY", + "contextParamConfigKey": "S3AsyncClientDecorator.MULTIPART_CONFIGURATION_KEY" }, "interceptors": [ "software.amazon.awssdk.services.s3.internal.handlers.PutObjectInterceptor", From 01c67875ccf7296183b105cc24318e20fd2327bf Mon Sep 17 00:00:00 2001 From: Olivier L Applin Date: Mon, 31 Jul 2023 18:56:36 -0400 Subject: [PATCH 08/10] javadoc --- .../s3/multipart/MultipartConfiguration.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java index acddfd02e91b..706eee550408 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -63,8 +63,7 @@ public Builder toBuilder() { /** * Indicates the value of the configured threshold, in bytes. Any request whose size is less than the configured value will - * not - * use multipart operation + * not use multipart operation * @return the value of the configured threshold. */ public Long thresholdInBytes() { @@ -73,9 +72,7 @@ public Long thresholdInBytes() { /** * Indicated the size, in bytes, of each individual part of the part requests. The actual part size used might be bigger to - * conforms to - * the maximum - * number of parts allowed per multipart requests. + * conforms to the maximum number of parts allowed per multipart requests. * @return the value of the configured part size. */ public Long minimumPartSizeInBytes() { @@ -100,7 +97,7 @@ public interface Builder extends CopyableBuilder

+ *

* * Default value: 8 Mib * @@ -117,17 +114,17 @@ public interface Builder extends CopyableBuilder

+ *

* When uploading large payload, the size of the payload of each individual part requests might actually be * bigger than * the configured value since there is a limit to the maximum number of parts possible per multipart request. If the * configured part size would lead to a number of parts higher than the maximum allowed, a larger part size will be * calculated instead to allow fewer part to be uploaded, to avoid the limit imposed on the maximum number of parts. - *

+ *

* In the case where the {@code minimumPartSizeInBytes} is set to a value higher than the {@code thresholdInBytes}, when * the client receive a request with a size smaller than a single part multipart operation will NOT be performed * even if the size of the request is larger than the threshold. - *

+ *

* Default value: 8 Mib * * @param minimumPartSizeInBytes the value of the part size to set @@ -144,7 +141,7 @@ public interface Builder extends CopyableBuilder

+ *

* Default value: If not specified, the SDK will use the equivalent of two parts worth of memory, so 16 Mib by default. * * @param apiCallBufferSizeInBytes the value of the maximum memory usage. From e4f8ab65e86ea1e6cab9f2ad75d6ff93e3f914b5 Mon Sep 17 00:00:00 2001 From: Olivier L Applin Date: Mon, 31 Jul 2023 18:59:50 -0400 Subject: [PATCH 09/10] more javadoc --- .../awssdk/services/s3/multipart/MultipartConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java index 706eee550408..f7eee1373ac2 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -114,6 +114,7 @@ public interface Builder extends CopyableBuilder * When uploading large payload, the size of the payload of each individual part requests might actually be * bigger than From c0a3c3f669d0886937b8173a70655dad6fc1f76c Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Aug 2023 10:58:28 -0400 Subject: [PATCH 10/10] Use 4x part size as default apiCallBufferSize --- .../s3/internal/multipart/MultipartS3AsyncClient.java | 4 ++-- .../awssdk/services/s3/multipart/MultipartConfiguration.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 5e706b50cd03..65b26ddec971 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -48,7 +48,7 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024; private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024; - private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 2; + private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 4; private final UploadObjectHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; @@ -68,7 +68,7 @@ private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mu } private long computeApiCallBufferSize(MultipartConfiguration multipartConfiguration) { - return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 2 + return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 4 : DEFAULT_API_CALL_BUFFER_SIZE; } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java index f7eee1373ac2..28e418974db8 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -143,7 +143,7 @@ public interface Builder extends CopyableBuilder - * Default value: If not specified, the SDK will use the equivalent of two parts worth of memory, so 16 Mib by default. + * Default value: If not specified, the SDK will use the equivalent of four parts worth of memory, so 32 Mib by default. * * @param apiCallBufferSizeInBytes the value of the maximum memory usage. * @return an instance of this builder.