Fixed the issue where thresholdInBytes is not the same as minimalPart…

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartConfigurationResolver.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.internal.multipart;
+
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
+import software.amazon.awssdk.utils.Validate;
+
+/**
+ * Internal utility class to resolve {@link MultipartConfiguration}.
+ */
+@SdkInternalApi
+public final class MultipartConfigurationResolver {
+
+    private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024;
+    private final long minimalPartSizeInBytes;
+    private final long apiCallBufferSize;
+    private final long thresholdInBytes;
+
+    public MultipartConfigurationResolver(MultipartConfiguration multipartConfiguration) {
+        Validate.notNull(multipartConfiguration, "multipartConfiguration");
+        this.minimalPartSizeInBytes = Validate.getOrDefault(multipartConfiguration.minimumPartSizeInBytes(),
+                                                            () -> DEFAULT_MIN_PART_SIZE);
+        this.apiCallBufferSize = Validate.getOrDefault(multipartConfiguration.apiCallBufferSizeInBytes(),
+                                                       () -> minimalPartSizeInBytes * 4);
+        this.thresholdInBytes = Validate.getOrDefault(multipartConfiguration.thresholdInBytes(), () -> minimalPartSizeInBytes);
+    }
+
+    public long minimalPartSizeInBytes() {
+        return minimalPartSizeInBytes;
+    }
+
+    public long thresholdInBytes() {
+        return thresholdInBytes;
+    }
+
+    public long apiCallBufferSize() {
+        return apiCallBufferSize;
+    }
+}

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java

@@ -46,32 +46,20 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient {
 
     private static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build();
 
-    private static final long DEFAULT_MIN_PART_SIZE = 8L * 1024 * 1024;
-    private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024;
-    private static final long DEFAULT_API_CALL_BUFFER_SIZE = DEFAULT_MIN_PART_SIZE * 4;
-
     private final UploadObjectHelper mpuHelper;
     private final CopyObjectHelper copyObjectHelper;
 
     private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) {
         super(delegate);
         MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration,
                                                                           MultipartConfiguration.builder()::build);
-        long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(),
-                                                        () -> DEFAULT_MIN_PART_SIZE);
-        long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(),
-                                               () -> DEFAULT_THRESHOLD);
-        long apiCallBufferSizeInBytes = Validate.getOrDefault(validConfiguration.apiCallBufferSizeInBytes(),
-                                                              () -> computeApiCallBufferSize(validConfiguration));
-        mpuHelper = new UploadObjectHelper(delegate, minPartSizeInBytes, threshold, apiCallBufferSizeInBytes);
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(validConfiguration);
+        long minPartSizeInBytes = resolver.minimalPartSizeInBytes();
+        long threshold = resolver.thresholdInBytes();
+        mpuHelper = new UploadObjectHelper(delegate, resolver);
         copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold);
     }
 
-    private long computeApiCallBufferSize(MultipartConfiguration multipartConfiguration) {
-        return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 4
-                                                                       : DEFAULT_API_CALL_BUFFER_SIZE;
-    }
-
     @Override
     public CompletableFuture<PutObjectResponse> putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) {
         return mpuHelper.uploadObject(putObjectRequest, requestBody);

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/UploadObjectHelper.java

@@ -34,30 +34,28 @@ public final class UploadObjectHelper {
     private final long partSizeInBytes;
     private final GenericMultipartHelper<PutObjectRequest, PutObjectResponse> genericMultipartHelper;
 
-    private final long maxMemoryUsageInBytes;
+    private final long apiCallBufferSize;
     private final long multipartUploadThresholdInBytes;
     private final UploadWithKnownContentLengthHelper uploadWithKnownContentLength;
     private final UploadWithUnknownContentLengthHelper uploadWithUnknownContentLength;
 
     public UploadObjectHelper(S3AsyncClient s3AsyncClient,
-                              long partSizeInBytes,
-                              long multipartUploadThresholdInBytes,
-                              long maxMemoryUsageInBytes) {
+                              MultipartConfigurationResolver resolver) {
         this.s3AsyncClient = s3AsyncClient;
-        this.partSizeInBytes = partSizeInBytes;
+        this.partSizeInBytes = resolver.minimalPartSizeInBytes();
         this.genericMultipartHelper = new GenericMultipartHelper<>(s3AsyncClient,
                                                                    SdkPojoConversionUtils::toAbortMultipartUploadRequest,
                                                                    SdkPojoConversionUtils::toPutObjectResponse);
-        this.maxMemoryUsageInBytes = maxMemoryUsageInBytes;
-        this.multipartUploadThresholdInBytes = multipartUploadThresholdInBytes;
+        this.apiCallBufferSize = resolver.apiCallBufferSize();
+        this.multipartUploadThresholdInBytes = resolver.thresholdInBytes();
         this.uploadWithKnownContentLength = new UploadWithKnownContentLengthHelper(s3AsyncClient,
                                                                                    partSizeInBytes,
                                                                                    multipartUploadThresholdInBytes,
-                                                                                   maxMemoryUsageInBytes);
+                                                                                   apiCallBufferSize);
         this.uploadWithUnknownContentLength = new UploadWithUnknownContentLengthHelper(s3AsyncClient,
                                                                                        partSizeInBytes,
                                                                                        multipartUploadThresholdInBytes,
-                                                                                       maxMemoryUsageInBytes);
+                                                                                       apiCallBufferSize);
     }
 
     public CompletableFuture<PutObjectResponse> uploadObject(PutObjectRequest putObjectRequest,

services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java

@@ -93,13 +93,19 @@ public Long apiCallBufferSizeInBytes() {
     public interface Builder extends CopyableBuilder<Builder, MultipartConfiguration> {
 
         /**
-         * Configures the minimum number of bytes of the body of the request required for requests to be converted to their
-         * multipart equivalent. Only taken into account when converting {@code putObject} and {@code copyObject} requests.
-         * Any request whose size is less than the  configured value will not use multipart operation,
-         * even if multipart is enabled via {@link S3AsyncClientBuilder#multipartEnabled(Boolean)}.
+         * Configure the size threshold, in bytes, for when to use multipart upload. Uploads/copies over this size will
+         * automatically use a multipart upload strategy, while uploads/copies smaller than this threshold will use a single
+         * connection to upload/copy the whole object.
+         *
          * <p>
+         * Multipart uploads are easier to recover from and also potentially faster than single part uploads, especially when the
+         * upload parts can be uploaded in parallel. Because there are additional network API calls, small objects are still
+         * recommended to use a single connection for the upload. See
+         * <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html">Uploading and copying objects using
+         * multipart upload</a>.
          *
-         * Default value: 8 Mib
+         * <p>
+         * By default, it is the same as {@link #minimumPartSizeInBytes(Long)}.
          *
          * @param thresholdInBytes the value of the threshold to set.
          * @return an instance of this builder.

services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartConfigurationResolverTest.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.internal.multipart;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
+
+public class MultipartConfigurationResolverTest {
+
+    @Test
+    void resolveThresholdInBytes_valueNotProvided_shouldSameAsPartSize() {
+        MultipartConfiguration configuration = MultipartConfiguration.builder()
+                                                                     .minimumPartSizeInBytes(10L)
+                                                                     .build();
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
+        assertThat(resolver.thresholdInBytes()).isEqualTo(10L);
+    }
+
+    @Test
+    void resolveThresholdInBytes_valueProvided_shouldHonor() {
+        MultipartConfiguration configuration = MultipartConfiguration.builder()
+                                                                     .minimumPartSizeInBytes(1L)
+                                                                     .thresholdInBytes(12L)
+                                                                     .build();
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
+        assertThat(resolver.thresholdInBytes()).isEqualTo(12L);
+    }
+
+    @Test
+    void resolveApiCallBufferSize_valueProvided_shouldHonor() {
+        MultipartConfiguration configuration = MultipartConfiguration.builder()
+                                                                     .apiCallBufferSizeInBytes(100L)
+                                                                     .build();
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
+        assertThat(resolver.apiCallBufferSize()).isEqualTo(100L);
+    }
+
+    @Test
+    void resolveApiCallBufferSize_valueNotProvided_shouldComputeBasedOnPartSize() {
+        MultipartConfiguration configuration = MultipartConfiguration.builder()
+                                                                     .minimumPartSizeInBytes(10L)
+                                                                     .build();
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
+        assertThat(resolver.apiCallBufferSize()).isEqualTo(40L);
+    }
+
+    @Test
+    void valueProvidedForAllFields_shouldHonor() {
+        MultipartConfiguration configuration = MultipartConfiguration.builder()
+                                                                     .minimumPartSizeInBytes(10L)
+                                                                     .thresholdInBytes(8L)
+                                                                     .apiCallBufferSizeInBytes(3L)
+                                                                     .build();
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(configuration);
+        assertThat(resolver.minimalPartSizeInBytes()).isEqualTo(10L);
+        assertThat(resolver.thresholdInBytes()).isEqualTo(8L);
+        assertThat(resolver.apiCallBufferSize()).isEqualTo(3L);
+    }
+
+    @Test
+    void noValueProvided_shouldUseDefault() {
+        MultipartConfigurationResolver resolver = new MultipartConfigurationResolver(MultipartConfiguration.builder()
+                                                                                                           .build());
+        assertThat(resolver.minimalPartSizeInBytes()).isEqualTo(8L * 1024 * 1024);
+        assertThat(resolver.thresholdInBytes()).isEqualTo(8L * 1024 * 1024);
+        assertThat(resolver.apiCallBufferSize()).isEqualTo(8L * 1024 * 1024 * 4);
+    }
+}

services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/UploadObjectHelperTest.java

@@ -61,6 +61,7 @@
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
 import software.amazon.awssdk.testutils.RandomTempFile;
 import software.amazon.awssdk.utils.CompletableFutureUtils;
 
@@ -97,7 +98,12 @@ public static Stream<AsyncRequestBody> asyncRequestBody() {
     @BeforeEach
     public void beforeEach() {
         s3AsyncClient = Mockito.mock(S3AsyncClient.class);
-        uploadHelper = new UploadObjectHelper(s3AsyncClient, PART_SIZE, THRESHOLD, PART_SIZE * 2);
+        uploadHelper = new UploadObjectHelper(s3AsyncClient,
+                                              new MultipartConfigurationResolver(MultipartConfiguration.builder()
+                                                                                                       .minimumPartSizeInBytes(PART_SIZE)
+                                                                                                       .thresholdInBytes(THRESHOLD)
+                                                                                                       .thresholdInBytes(PART_SIZE * 2)
+                                                                                                       .build()));
     }
 
     @ParameterizedTest