diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index 596d44bcf14b..0bef67df7867 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -227,6 +227,11 @@ public class CustomizationConfig { */ private String asyncClientDecorator; + /** + * Only for s3. A set of customization to related to multipart operations. + */ + private MultipartCustomization multipartCustomization; + /** * Whether to skip generating endpoint tests from endpoint-tests.json */ @@ -665,4 +670,12 @@ public Map getCustomClientContextParams() { public void setCustomClientContextParams(Map customClientContextParams) { this.customClientContextParams = customClientContextParams; } + + public MultipartCustomization getMultipartCustomization() { + return this.multipartCustomization; + } + + public void setMultipartCustomization(MultipartCustomization multipartCustomization) { + this.multipartCustomization = multipartCustomization; + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java new file mode 100644 index 000000000000..94264a9e5ec6 --- /dev/null +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.model.config.customization; + +public class MultipartCustomization { + private String multipartConfigurationClass; + private String multipartConfigMethodDoc; + private String multipartEnableMethodDoc; + private String contextParamEnabledKey; + private String contextParamConfigKey; + + public String getMultipartConfigurationClass() { + return multipartConfigurationClass; + } + + public void setMultipartConfigurationClass(String multipartConfigurationClass) { + this.multipartConfigurationClass = multipartConfigurationClass; + } + + public String getMultipartConfigMethodDoc() { + return multipartConfigMethodDoc; + } + + public void setMultipartConfigMethodDoc(String multipartMethodDoc) { + this.multipartConfigMethodDoc = multipartMethodDoc; + } + + public String getMultipartEnableMethodDoc() { + return multipartEnableMethodDoc; + } + + public void setMultipartEnableMethodDoc(String multipartEnableMethodDoc) { + this.multipartEnableMethodDoc = multipartEnableMethodDoc; + } + + public String getContextParamEnabledKey() { + return contextParamEnabledKey; + } + + public void setContextParamEnabledKey(String contextParamEnabledKey) { + this.contextParamEnabledKey = contextParamEnabledKey; + } + + public String getContextParamConfigKey() { + return contextParamConfigKey; + } + + public void setContextParamConfigKey(String contextParamConfigKey) { + this.contextParamConfigKey = contextParamConfigKey; + } +} diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java index a8265f0dc7f1..59a719fb2c7d 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java @@ -20,7 +20,7 @@ import java.util.Collections; /** - * Represents the a Poet generated class + * Represents a Poet generated class */ public interface ClassSpec { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java index 509a30c6c8d7..3ff2b99ec98e 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java @@ -17,6 +17,7 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; import java.net.URI; @@ -24,6 +25,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.auth.token.credentials.SdkTokenProvider; import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetExtension; @@ -59,12 +61,12 @@ public AsyncClientBuilderClass(IntermediateModel model) { @Override public TypeSpec poetSpec() { TypeSpec.Builder builder = - PoetUtils.createClassBuilder(builderClassName) - .addAnnotation(SdkInternalApi.class) - .addModifiers(Modifier.FINAL) - .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) - .addSuperinterface(builderInterfaceName) - .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); + PoetUtils.createClassBuilder(builderClassName) + .addAnnotation(SdkInternalApi.class) + .addModifiers(Modifier.FINAL) + .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) + .addSuperinterface(builderInterfaceName) + .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); if (model.getEndpointOperation().isPresent()) { builder.addMethod(endpointDiscoveryEnabled()); @@ -80,6 +82,12 @@ public TypeSpec poetSpec() { builder.addMethod(bearerTokenProviderMethod()); } + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + builder.addMethod(multipartEnabledMethod(multipartCustomization)); + builder.addMethod(multipartConfigMethods(multipartCustomization)); + } + builder.addMethod(buildClientMethod()); builder.addMethod(initializeServiceClientConfigMethod()); @@ -124,15 +132,15 @@ private MethodSpec endpointProviderMethod() { private MethodSpec buildClientMethod() { MethodSpec.Builder builder = MethodSpec.methodBuilder("buildClient") - .addAnnotation(Override.class) - .addModifiers(Modifier.PROTECTED, Modifier.FINAL) - .returns(clientInterfaceName) - .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", - SdkClientConfiguration.class).addStatement("this.validateClientOptions" - + "(clientConfiguration)") - .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" - + "(clientConfiguration)", - serviceConfigClassName); + .addAnnotation(Override.class) + .addModifiers(Modifier.PROTECTED, Modifier.FINAL) + .returns(clientInterfaceName) + .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", + SdkClientConfiguration.class) + .addStatement("this.validateClientOptions(clientConfiguration)") + .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" + + "(clientConfiguration)", + serviceConfigClassName); builder.addStatement("$1T client = new $2T(serviceClientConfiguration, clientConfiguration)", clientInterfaceName, clientClassName); @@ -156,6 +164,32 @@ private MethodSpec bearerTokenProviderMethod() { .build(); } + private MethodSpec multipartEnabledMethod(MultipartCustomization multipartCustomization) { + return MethodSpec.methodBuilder("multipartEnabled") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addStatement("clientContextParams.put($N, enabled)", + multipartCustomization.getContextParamEnabledKey()) + .addStatement("return this") + .build(); + } + + private MethodSpec multipartConfigMethods(MultipartCustomization multipartCustomization) { + ClassName mulitpartConfigClassName = + PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); + return MethodSpec.methodBuilder("multipartConfiguration") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfig").build()) + .returns(builderInterfaceName) + .addStatement("clientContextParams.put($N, multipartConfig)", + multipartCustomization.getContextParamConfigKey()) + .addStatement("return this") + .build(); + } + private MethodSpec initializeServiceClientConfigMethod() { return MethodSpec.methodBuilder("initializeServiceClientConfig").addModifiers(Modifier.PRIVATE) .addParameter(SdkClientConfiguration.class, "clientConfig") diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java index 5348972b5df9..df62f97ae7c0 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java @@ -17,34 +17,97 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.CodeBlock; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; +import java.util.function.Consumer; +import javax.lang.model.element.Modifier; import software.amazon.awssdk.awscore.client.builder.AwsAsyncClientBuilder; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; public class AsyncClientBuilderInterface implements ClassSpec { + private static final Logger log = Logger.loggerFor(AsyncClientBuilderInterface.class); + private final ClassName builderInterfaceName; private final ClassName clientInterfaceName; private final ClassName baseBuilderInterfaceName; + private final IntermediateModel model; public AsyncClientBuilderInterface(IntermediateModel model) { String basePackage = model.getMetadata().getFullClientPackageName(); this.clientInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncInterface()); this.builderInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncBuilderInterface()); this.baseBuilderInterfaceName = ClassName.get(basePackage, model.getMetadata().getBaseBuilderInterface()); + this.model = model; } @Override public TypeSpec poetSpec() { - return PoetUtils.createInterfaceBuilder(builderInterfaceName) - .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), - builderInterfaceName, clientInterfaceName)) - .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, - builderInterfaceName, clientInterfaceName)) - .addJavadoc(getJavadoc()) - .build(); + TypeSpec.Builder builder = PoetUtils + .createInterfaceBuilder(builderInterfaceName) + .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), + builderInterfaceName, clientInterfaceName)) + .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, + builderInterfaceName, clientInterfaceName)) + .addJavadoc(getJavadoc()); + + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + includeMultipartMethod(builder, multipartCustomization); + } + return builder.build(); + } + + private void includeMultipartMethod(TypeSpec.Builder builder, MultipartCustomization multipartCustomization) { + log.debug(() -> String.format("Adding multipart config methods to builder interface for service '%s'", + model.getMetadata().getServiceId())); + + // .multipartEnabled(Boolean) + builder.addMethod( + MethodSpec.methodBuilder("multipartEnabled") + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartEnableMethodDoc())) + .build()); + + // .multipartConfiguration(MultipartConfiguration) + String multiPartConfigMethodName = "multipartConfiguration"; + String multipartConfigClass = Validate.notNull(multipartCustomization.getMultipartConfigurationClass(), + "'multipartConfigurationClass' must be defined"); + ClassName mulitpartConfigClassName = PoetUtils.classNameFromFqcn(multipartConfigClass); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfiguration").build()) + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); + + // .multipartConfiguration(Consumer) + ClassName mulitpartConfigBuilderClassName = PoetUtils.classNameFromFqcn(multipartConfigClass + ".Builder"); + ParameterizedTypeName consumerBuilderType = ParameterizedTypeName.get(ClassName.get(Consumer.class), + mulitpartConfigBuilderClassName); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(consumerBuilderType, "multipartConfiguration").build()) + .addStatement("$T builder = $T.builder()", + mulitpartConfigBuilderClassName, + mulitpartConfigClassName) + .addStatement("multipartConfiguration.accept(builder)") + .addStatement("return multipartConfiguration(builder.build())") + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); } @Override diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java index 63dcf2ddc88f..03cf42afe5df 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java @@ -117,7 +117,7 @@ protected static void deleteBucketAndAllContents(String bucketName) { S3TestUtils.deleteBucketAndAllContents(s3, bucketName); } - private static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { + protected static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { private final String clientName; private final ClientType clientType; diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java index 6db434526fb9..fc4f31b76b1a 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java @@ -31,17 +31,16 @@ import javax.crypto.KeyGenerator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.MetadataDirective; @@ -58,6 +57,7 @@ public class S3ClientMultiPartCopyIntegrationTest extends S3IntegrationTestBase private static final long SMALL_OBJ_SIZE = 1024 * 1024; private static S3AsyncClient s3CrtAsyncClient; private static S3AsyncClient s3MpuClient; + @BeforeAll public static void setUp() throws Exception { S3IntegrationTestBase.setUp(); @@ -66,7 +66,13 @@ public static void setUp() throws Exception { .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) .build(); - s3MpuClient = new MultipartS3AsyncClient(s3Async); + s3MpuClient = S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -158,7 +164,7 @@ private static byte[] generateSecretKey() { private void createOriginalObject(byte[] originalContent, String originalKey) { s3CrtAsyncClient.putObject(r -> r.bucket(BUCKET) - .key(originalKey), + .key(originalKey), AsyncRequestBody.fromBytes(originalContent)).join(); } diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java index cb72906943b9..fa31b5453e5e 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java @@ -15,7 +15,6 @@ package software.amazon.awssdk.services.s3.multipart; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; @@ -27,25 +26,21 @@ import java.nio.file.Files; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.reactivestreams.Subscriber; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.internal.async.FileAsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.utils.ChecksumUtils; -import software.amazon.awssdk.testutils.RandomTempFile; @Timeout(value = 30, unit = SECONDS) public class S3MultipartClientPutObjectIntegrationTest extends S3IntegrationTestBase { @@ -66,7 +61,14 @@ public static void setup() throws Exception { testFile = File.createTempFile("SplittingPublisherTest", UUID.randomUUID().toString()); Files.write(testFile.toPath(), CONTENT); - mpuS3Client = new MultipartS3AsyncClient(s3Async); + mpuS3Client = S3AsyncClient + .builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -81,8 +83,9 @@ void putObject_fileRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); mpuS3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(testFile.length()); byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); @@ -95,8 +98,9 @@ void putObject_byteAsyncRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromBytes(bytes); mpuS3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(OBJ_SIZE); byte[] expectedSum = ChecksumUtils.computeCheckSum(new ByteArrayInputStream(bytes)); @@ -120,8 +124,9 @@ public void subscribe(Subscriber s) { } }).get(30, SECONDS); - ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); assertThat(objContent.response().contentLength()).isEqualTo(testFile.length()); byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java index 2dbb61091da2..b751cb29c1b0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java @@ -23,11 +23,17 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.endpoints.S3ClientContextParams; import software.amazon.awssdk.services.s3.internal.crossregion.S3CrossRegionAsyncClient; +import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.ConditionalDecorator; @SdkInternalApi public class S3AsyncClientDecorator { + public static final AttributeMap.Key MULTIPART_CONFIGURATION_KEY = + new AttributeMap.Key(MultipartConfiguration.class){}; + public static final AttributeMap.Key MULTIPART_ENABLED_KEY = + new AttributeMap.Key(Boolean.class){}; public S3AsyncClientDecorator() { } @@ -36,14 +42,26 @@ public S3AsyncClient decorate(S3AsyncClient base, SdkClientConfiguration clientConfiguration, AttributeMap clientContextParams) { List> decorators = new ArrayList<>(); - decorators.add(ConditionalDecorator.create(isCrossRegionEnabledAsync(clientContextParams), - S3CrossRegionAsyncClient::new)); + decorators.add(ConditionalDecorator.create( + isCrossRegionEnabledAsync(clientContextParams), + S3CrossRegionAsyncClient::new)); + + decorators.add(ConditionalDecorator.create( + isMultipartEnable(clientContextParams), + client -> { + MultipartConfiguration multipartConfiguration = clientContextParams.get(MULTIPART_CONFIGURATION_KEY); + return MultipartS3AsyncClient.create(client, multipartConfiguration); + })); return ConditionalDecorator.decorate(base, decorators); } private Predicate isCrossRegionEnabledAsync(AttributeMap clientContextParams) { Boolean crossRegionEnabled = clientContextParams.get(S3ClientContextParams.CROSS_REGION_ACCESS_ENABLED); - return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); + return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); } + private Predicate isMultipartEnable(AttributeMap clientContextParams) { + Boolean multipartEnabled = clientContextParams.get(MULTIPART_ENABLED_KEY); + return client -> multipartEnabled != null && multipartEnabled.booleanValue(); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java index 31b947bb89c5..16294ff8f065 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java @@ -24,8 +24,6 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.internal.crt.UploadPartCopyRequestIterable; -import software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper; -import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; @@ -130,6 +128,10 @@ private void doCopyInParts(CopyObjectRequest copyObjectRequest, long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); + if (optimalPartSize > partSizeInBytes) { + log.debug(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } log.debug(() -> String.format("Starting multipart copy with partCount: %s, optimalPartSize: %s", partCount, optimalPartSize)); @@ -170,7 +172,6 @@ private CompletableFuture completeMultipartUplo .parts(parts) .build()) .build(); - return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -201,7 +202,8 @@ private void sendIndividualUploadPartCopy(String uploadId, log.debug(() -> "Sending uploadPartCopyRequest with range: " + uploadPartCopyRequest.copySourceRange() + " uploadId: " + uploadId); - CompletableFuture uploadPartCopyFuture = s3AsyncClient.uploadPartCopy(uploadPartCopyRequest); + CompletableFuture uploadPartCopyFuture = + s3AsyncClient.uploadPartCopy(uploadPartCopyRequest); CompletableFuture convertFuture = uploadPartCopyFuture.thenApply(uploadPartCopyResponse -> diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java index 905c1bc928ea..38e76394958e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/GenericMultipartHelper.java @@ -91,7 +91,6 @@ public CompletableFuture completeMultipartUploa .parts(parts) .build()) .build(); - return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); } @@ -125,7 +124,8 @@ public BiFunction handleExcept public void cleanUpParts(String uploadId, AbortMultipartUploadRequest.Builder abortMultipartUploadRequest) { log.debug(() -> "Aborting multipart upload: " + uploadId); - s3AsyncClient.abortMultipartUpload(abortMultipartUploadRequest.uploadId(uploadId).build()) + AbortMultipartUploadRequest request = abortMultipartUploadRequest.uploadId(uploadId).build(); + s3AsyncClient.abortMultipartUpload(request) .exceptionally(throwable -> { log.warn(() -> String.format("Failed to abort previous multipart upload " + "(id: %s)" diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index a4b3147254f9..65b26ddec971 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -17,31 +17,59 @@ import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Request; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.utils.Validate; -// This is just a temporary class for testing -//TODO: change this +/** + * An {@link S3AsyncClient} that automatically converts put, copy requests to their respective multipart call. Note: get is not + * yet supported. + * + * @see MultipartConfiguration + */ @SdkInternalApi -public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { - private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; +public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { + + private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); + + private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024; private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024; + private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 4; - private static final long DEFAULT_MAX_MEMORY = DEFAULT_PART_SIZE_IN_BYTES * 2; private final UploadObjectHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; - public MultipartS3AsyncClient(S3AsyncClient delegate) { + private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) { super(delegate); - // TODO: pass a config object to the upload helper instead - mpuHelper = new UploadObjectHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD, DEFAULT_MAX_MEMORY); - copyObjectHelper = new CopyObjectHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD); + MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration, + MultipartConfiguration.builder()::build); + long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(), + () -> DEFAULT_MIN_PART_SIZE); + long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(), + () -> DEFAULT_THRESHOLD); + long apiCallBufferSizeInBytes = Validate.getOrDefault(validConfiguration.apiCallBufferSizeInBytes(), + () -> computeApiCallBufferSize(validConfiguration)); + mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, apiCallBufferSizeInBytes); + copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); + } + + private long computeApiCallBufferSize(MultipartConfiguration multipartConfiguration) { + return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 4 + : DEFAULT_API_CALL_BUFFER_SIZE; } @Override @@ -54,8 +82,27 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb return copyObjectHelper.copyObject(copyObjectRequest); } + @Override + public CompletableFuture getObject( + GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { + throw new UnsupportedOperationException( + "Multipart download is not yet supported. Instead use the CRT based S3 client for multipart download."); + } + @Override public void close() { delegate().close(); } + + public static MultipartS3AsyncClient create(S3AsyncClient client, MultipartConfiguration multipartConfiguration) { + S3AsyncClient clientWithUserAgent = new DelegatingS3AsyncClient(client) { + @Override + protected CompletableFuture invokeOperation(T request, Function> operation) { + T requestWithUserAgent = UserAgentUtils.applyUserAgentInfo(request, c -> c.addApiName(USER_AGENT_API_NAME)); + return operation.apply(requestWithUserAgent); + } + }; + return new MultipartS3AsyncClient(clientWithUserAgent, multipartConfiguration); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index 1228e577fcd1..9754d284f5b9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -36,8 +36,8 @@ import software.amazon.awssdk.utils.Pair; /** - * A base class contains common logic used by {@link UploadWithUnknownContentLengthHelper} - * and {@link UploadWithKnownContentLengthHelper}. + * A base class contains common logic used by {@link UploadWithUnknownContentLengthHelper} and + * {@link UploadWithKnownContentLengthHelper}. */ @SdkInternalApi public final class MultipartUploadHelper { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java index e8bef01ab81b..a00b9eb9189d 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadWithKnownContentLengthHelper.java @@ -112,6 +112,10 @@ private void doUploadInParts(Pair request, long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); + if (optimalPartSize > partSizeInBytes) { + log.debug(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } log.debug(() -> String.format("Starting multipart upload with partCount: %d, optimalPartSize: %d", partCount, optimalPartSize)); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java new file mode 100644 index 000000000000..28e418974db8 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -0,0 +1,199 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.multipart; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Class that hold configuration properties related to multipart operation for a {@link S3AsyncClient}. Passing this class to the + * {@link S3AsyncClientBuilder#multipartConfiguration(MultipartConfiguration)} will enable automatic conversion of + * {@link S3AsyncClient#putObject(Consumer, AsyncRequestBody)}, {@link S3AsyncClient#copyObject(CopyObjectRequest)} to their + * respective multipart operation. + *

+ * Note: The multipart operation for {@link S3AsyncClient#getObject(GetObjectRequest, AsyncResponseTransformer)} is + * temporarily disabled and will result in throwing a {@link UnsupportedOperationException} if called when configured for + * multipart operation. + */ +@SdkPublicApi +public final class MultipartConfiguration implements ToCopyableBuilder { + + private final Long thresholdInBytes; + private final Long minimumPartSizeInBytes; + private final Long apiCallBufferSizeInBytes; + + private MultipartConfiguration(DefaultMultipartConfigBuilder builder) { + this.thresholdInBytes = builder.thresholdInBytes; + this.minimumPartSizeInBytes = builder.minimumPartSizeInBytes; + this.apiCallBufferSizeInBytes = builder.apiCallBufferSizeInBytes; + } + + public static Builder builder() { + return new DefaultMultipartConfigBuilder(); + } + + @Override + public Builder toBuilder() { + return builder() + .apiCallBufferSizeInBytes(apiCallBufferSizeInBytes) + .minimumPartSizeInBytes(minimumPartSizeInBytes) + .thresholdInBytes(thresholdInBytes); + } + + /** + * Indicates the value of the configured threshold, in bytes. Any request whose size is less than the configured value will + * not use multipart operation + * @return the value of the configured threshold. + */ + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + /** + * Indicated the size, in bytes, of each individual part of the part requests. The actual part size used might be bigger to + * conforms to the maximum number of parts allowed per multipart requests. + * @return the value of the configured part size. + */ + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + /** + * The maximum memory, in bytes, that the SDK will use to buffer requests content into memory. + * @return the value of the configured maximum memory usage. + */ + public Long apiCallBufferSizeInBytes() { + return this.apiCallBufferSizeInBytes; + } + + /** + * Builder for a {@link MultipartConfiguration}. + */ + public interface Builder extends CopyableBuilder { + + /** + * Configures the minimum number of bytes of the body of the request required for requests to be converted to their + * multipart equivalent. Only taken into account when converting {@code putObject} and {@code copyObject} requests. + * Any request whose size is less than the configured value will not use multipart operation, + * even if multipart is enabled via {@link S3AsyncClientBuilder#multipartEnabled(Boolean)}. + *

+ * + * Default value: 8 Mib + * + * @param thresholdInBytes the value of the threshold to set. + * @return an instance of this builder. + */ + Builder thresholdInBytes(Long thresholdInBytes); + + /** + * Indicates the value of the configured threshold. + * @return the value of the threshold. + */ + Long thresholdInBytes(); + + /** + * Configures the part size, in bytes, to be used in each individual part requests. + * Only used for putObject and copyObject operations. + *

+ * When uploading large payload, the size of the payload of each individual part requests might actually be + * bigger than + * the configured value since there is a limit to the maximum number of parts possible per multipart request. If the + * configured part size would lead to a number of parts higher than the maximum allowed, a larger part size will be + * calculated instead to allow fewer part to be uploaded, to avoid the limit imposed on the maximum number of parts. + *

+ * In the case where the {@code minimumPartSizeInBytes} is set to a value higher than the {@code thresholdInBytes}, when + * the client receive a request with a size smaller than a single part multipart operation will NOT be performed + * even if the size of the request is larger than the threshold. + *

+ * Default value: 8 Mib + * + * @param minimumPartSizeInBytes the value of the part size to set + * @return an instance of this builder. + */ + Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes); + + /** + * Indicated the value of the part configured size. + * @return the value of the part size + */ + Long minimumPartSizeInBytes(); + + /** + * Configures the maximum amount of memory, in bytes, the SDK will use to buffer content of requests in memory. + * Increasing this value may lead to better performance at the cost of using more memory. + *

+ * Default value: If not specified, the SDK will use the equivalent of four parts worth of memory, so 32 Mib by default. + * + * @param apiCallBufferSizeInBytes the value of the maximum memory usage. + * @return an instance of this builder. + */ + Builder apiCallBufferSizeInBytes(Long apiCallBufferSizeInBytes); + + /** + * Indicates the value of the maximum memory usage that the SDK will use. + * @return the value of the maximum memory usage. + */ + Long apiCallBufferSizeInBytes(); + } + + private static class DefaultMultipartConfigBuilder implements Builder { + private Long thresholdInBytes; + private Long minimumPartSizeInBytes; + private Long apiCallBufferSizeInBytes; + + public Builder thresholdInBytes(Long thresholdInBytes) { + this.thresholdInBytes = thresholdInBytes; + return this; + } + + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + public Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes) { + this.minimumPartSizeInBytes = minimumPartSizeInBytes; + return this; + } + + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + @Override + public Builder apiCallBufferSizeInBytes(Long maximumMemoryUsageInBytes) { + this.apiCallBufferSizeInBytes = maximumMemoryUsageInBytes; + return this; + } + + @Override + public Long apiCallBufferSizeInBytes() { + return apiCallBufferSizeInBytes; + } + + @Override + public MultipartConfiguration build() { + return new MultipartConfiguration(this); + } + } +} diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 1a1efb76c5f4..ccddba62880c 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -236,6 +236,13 @@ "syncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3SyncClientDecorator", "asyncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3AsyncClientDecorator", "useGlobalEndpoint": true, + "multipartCustomization": { + "multipartConfigurationClass": "software.amazon.awssdk.services.s3.multipart.MultipartConfiguration", + "multipartConfigMethodDoc": "Configuration for multipart operation of this client.", + "multipartEnableMethodDoc": "Enables automatic conversion of put and copy method to their equivalent multipart operation.", + "contextParamEnabledKey": "S3AsyncClientDecorator.MULTIPART_ENABLED_KEY", + "contextParamConfigKey": "S3AsyncClientDecorator.MULTIPART_CONFIGURATION_KEY" + }, "interceptors": [ "software.amazon.awssdk.services.s3.internal.handlers.PutObjectInterceptor", "software.amazon.awssdk.services.s3.internal.handlers.CreateBucketInterceptor", diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java new file mode 100644 index 000000000000..0f41c7c78e74 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartClientUserAgentTest.java @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; + +class MultipartClientUserAgentTest { + private MockAsyncHttpClient mockAsyncHttpClient; + private UserAgentInterceptor userAgentInterceptor; + private S3AsyncClient s3Client; + + @BeforeEach + void init() { + this.mockAsyncHttpClient = new MockAsyncHttpClient(); + this.userAgentInterceptor = new UserAgentInterceptor(); + s3Client = S3AsyncClient.builder() + .httpClient(mockAsyncHttpClient) + .endpointOverride(URI.create("http://localhost")) + .overrideConfiguration(c -> c.addExecutionInterceptor(userAgentInterceptor)) + .multipartConfiguration(c -> c.minimumPartSizeInBytes(512L).thresholdInBytes(512L)) + .multipartEnabled(true) + .region(Region.US_EAST_1) + .build(); + } + + @AfterEach + void reset() { + this.mockAsyncHttpClient.reset(); + } + + @Test + void validateUserAgent_nonMultipartMethod() throws Exception { + HttpExecuteResponse response = HttpExecuteResponse.builder() + .response(SdkHttpResponse.builder().statusCode(200).build()) + .build(); + mockAsyncHttpClient.stubResponses(response); + + s3Client.headObject(req -> req.key("mock").bucket("mock")).get(); + + assertThat(userAgentInterceptor.apiNames) + .anyMatch(api -> "hll".equals(api.name()) && "s3Multipart".equals(api.version())); + } + + private static final class UserAgentInterceptor implements ExecutionInterceptor { + private final List apiNames = new ArrayList<>(); + + @Override + public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes executionAttributes) { + context.request().overrideConfiguration().ifPresent(c -> apiNames.addAll(c.apiNames())); + } + } + +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java new file mode 100644 index 000000000000..510d441c4caa --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/S3MultipartClientBuilderTest.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; + +class S3MultipartClientBuilderTest { + + @Test + void multipartEnabledWithConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .multipartConfiguration(MultipartConfiguration.builder().build()) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartEnabledWithoutConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartDisabledWithConfig_shouldNotBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(false) + .multipartConfiguration(b -> b.apiCallBufferSizeInBytes(1024L)) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void noMultipart_shouldNotBeMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } +}