diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CopyObjectHelper.java index 414262b7bffa..9070eb7192c5 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CopyObjectHelper.java @@ -24,6 +24,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper; +import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; @@ -54,8 +55,8 @@ public CopyObjectHelper(S3AsyncClient s3AsyncClient, long partSizeInBytes) { this.s3AsyncClient = s3AsyncClient; this.partSizeInBytes = partSizeInBytes; this.genericMultipartHelper = new GenericMultipartHelper<>(s3AsyncClient, - RequestConversionUtils::toAbortMultipartUploadRequest, - RequestConversionUtils::toCopyObjectResponse); + SdkPojoConversionUtils::toAbortMultipartUploadRequest, + SdkPojoConversionUtils::toCopyObjectResponse); } public CompletableFuture copyObject(CopyObjectRequest copyObjectRequest) { @@ -64,7 +65,7 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb try { CompletableFuture headFuture = - s3AsyncClient.headObject(RequestConversionUtils.toHeadObjectRequest(copyObjectRequest)); + s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest)); // Ensure cancellations are forwarded to the head future CompletableFutureUtils.forwardExceptionTo(returnFuture, headFuture); @@ -101,7 +102,7 @@ private void copyInParts(CopyObjectRequest copyObjectRequest, Long contentLength, CompletableFuture returnFuture) { - CreateMultipartUploadRequest request = RequestConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest); + CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -212,7 +213,7 @@ private static CompletedPart convertUploadPartCopyResponse(AtomicReferenceArray< UploadPartCopyResponse uploadPartCopyResponse) { CopyPartResult copyPartResult = uploadPartCopyResponse.copyPartResult(); CompletedPart completedPart = - RequestConversionUtils.toCompletedPart(copyPartResult, + SdkPojoConversionUtils.toCompletedPart(copyPartResult, partNumber); completedParts.set(partNumber - 1, completedPart); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/RequestConversionUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/RequestConversionUtils.java deleted file mode 100644 index f4a3aaf60d4a..000000000000 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/RequestConversionUtils.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.internal.crt; - -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CopyObjectRequest; -import software.amazon.awssdk.services.s3.model.CopyObjectResponse; -import software.amazon.awssdk.services.s3.model.CopyPartResult; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; -import software.amazon.awssdk.services.s3.model.UploadPartResponse; - -/** - * Request conversion utility method for POJO classes associated with multipart feature. - */ -//TODO: iterate over SDK fields to get the data -@SdkInternalApi -public final class RequestConversionUtils { - - private RequestConversionUtils() { - } - - public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(PutObjectRequest putObjectRequest) { - - return CreateMultipartUploadRequest.builder() - .bucket(putObjectRequest.bucket()) - .key(putObjectRequest.key()) - .sseCustomerAlgorithm(putObjectRequest.sseCustomerAlgorithm()) - .sseCustomerKey(putObjectRequest.sseCustomerKey()) - .sseCustomerKeyMD5(putObjectRequest.sseCustomerKeyMD5()) - .requestPayer(putObjectRequest.requestPayer()) - .acl(putObjectRequest.acl()) - .cacheControl(putObjectRequest.cacheControl()) - .metadata(putObjectRequest.metadata()) - .contentDisposition(putObjectRequest.contentDisposition()) - .contentEncoding(putObjectRequest.contentEncoding()) - .contentType(putObjectRequest.contentType()) - .contentLanguage(putObjectRequest.contentLanguage()) - .grantFullControl(putObjectRequest.grantFullControl()) - .expires(putObjectRequest.expires()) - .grantRead(putObjectRequest.grantRead()) - .grantFullControl(putObjectRequest.grantFullControl()) - .grantReadACP(putObjectRequest.grantReadACP()) - .grantWriteACP(putObjectRequest.grantWriteACP()) - //TODO filter out headers - //.overrideConfiguration(putObjectRequest.overrideConfiguration()) - .build(); - } - - public static HeadObjectRequest toHeadObjectRequest(CopyObjectRequest copyObjectRequest) { - return HeadObjectRequest.builder() - .bucket(copyObjectRequest.sourceBucket()) - .key(copyObjectRequest.sourceKey()) - .versionId(copyObjectRequest.sourceVersionId()) - .ifMatch(copyObjectRequest.copySourceIfMatch()) - .ifModifiedSince(copyObjectRequest.copySourceIfModifiedSince()) - .ifNoneMatch(copyObjectRequest.copySourceIfNoneMatch()) - .ifUnmodifiedSince(copyObjectRequest.copySourceIfUnmodifiedSince()) - .expectedBucketOwner(copyObjectRequest.expectedSourceBucketOwner()) - .sseCustomerAlgorithm(copyObjectRequest.copySourceSSECustomerAlgorithm()) - .sseCustomerKey(copyObjectRequest.copySourceSSECustomerKey()) - .sseCustomerKeyMD5(copyObjectRequest.copySourceSSECustomerKeyMD5()) - .build(); - } - - public static CompletedPart toCompletedPart(CopyPartResult copyPartResult, int partNumber) { - return CompletedPart.builder() - .partNumber(partNumber) - .eTag(copyPartResult.eTag()) - .checksumCRC32C(copyPartResult.checksumCRC32C()) - .checksumCRC32(copyPartResult.checksumCRC32()) - .checksumSHA1(copyPartResult.checksumSHA1()) - .checksumSHA256(copyPartResult.checksumSHA256()) - .eTag(copyPartResult.eTag()) - .build(); - } - - public static CompletedPart toCompletedPart(UploadPartResponse partResponse, int partNumber) { - return CompletedPart.builder() - .partNumber(partNumber) - .eTag(partResponse.eTag()) - .checksumCRC32C(partResponse.checksumCRC32C()) - .checksumCRC32(partResponse.checksumCRC32()) - .checksumSHA1(partResponse.checksumSHA1()) - .checksumSHA256(partResponse.checksumSHA256()) - .eTag(partResponse.eTag()) - .build(); - } - - public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(CopyObjectRequest copyObjectRequest) { - return CreateMultipartUploadRequest.builder() - .bucket(copyObjectRequest.destinationBucket()) - .contentEncoding(copyObjectRequest.contentEncoding()) - .checksumAlgorithm(copyObjectRequest.checksumAlgorithmAsString()) - .tagging(copyObjectRequest.tagging()) - .contentType(copyObjectRequest.contentType()) - .contentLanguage(copyObjectRequest.contentLanguage()) - .contentDisposition(copyObjectRequest.contentDisposition()) - .cacheControl(copyObjectRequest.cacheControl()) - .expires(copyObjectRequest.expires()) - .key(copyObjectRequest.destinationKey()) - .websiteRedirectLocation(copyObjectRequest.websiteRedirectLocation()) - .expectedBucketOwner(copyObjectRequest.expectedBucketOwner()) - .requestPayer(copyObjectRequest.requestPayerAsString()) - .acl(copyObjectRequest.aclAsString()) - .grantRead(copyObjectRequest.grantRead()) - .grantReadACP(copyObjectRequest.grantReadACP()) - .grantWriteACP(copyObjectRequest.grantWriteACP()) - .grantFullControl(copyObjectRequest.grantFullControl()) - .storageClass(copyObjectRequest.storageClassAsString()) - .ssekmsKeyId(copyObjectRequest.ssekmsKeyId()) - .sseCustomerKey(copyObjectRequest.sseCustomerKey()) - .sseCustomerAlgorithm(copyObjectRequest.sseCustomerAlgorithm()) - .sseCustomerKeyMD5(copyObjectRequest.sseCustomerKeyMD5()) - .ssekmsEncryptionContext(copyObjectRequest.ssekmsEncryptionContext()) - .serverSideEncryption(copyObjectRequest.serverSideEncryptionAsString()) - .bucketKeyEnabled(copyObjectRequest.bucketKeyEnabled()) - .objectLockMode(copyObjectRequest.objectLockModeAsString()) - .objectLockLegalHoldStatus(copyObjectRequest.objectLockLegalHoldStatusAsString()) - .objectLockRetainUntilDate(copyObjectRequest.objectLockRetainUntilDate()) - .metadata(copyObjectRequest.metadata()) - .build(); - } - - public static CopyObjectResponse toCopyObjectResponse(CompleteMultipartUploadResponse response) { - CopyObjectResponse.Builder builder = CopyObjectResponse.builder() - .versionId(response.versionId()) - .copyObjectResult(b -> b.checksumCRC32(response.checksumCRC32()) - .checksumSHA1(response.checksumSHA1()) - .checksumSHA256(response.checksumSHA256()) - .checksumCRC32C(response.checksumCRC32C()) - .eTag(response.eTag()) - .build()) - .expiration(response.expiration()) - .bucketKeyEnabled(response.bucketKeyEnabled()) - .serverSideEncryption(response.serverSideEncryption()) - .ssekmsKeyId(response.ssekmsKeyId()) - .serverSideEncryption(response.serverSideEncryptionAsString()) - .requestCharged(response.requestChargedAsString()); - if (response.responseMetadata() != null) { - builder.responseMetadata(response.responseMetadata()); - } - - if (response.sdkHttpResponse() != null) { - builder.sdkHttpResponse(response.sdkHttpResponse()); - } - - return builder.build(); - } - - public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(CopyObjectRequest copyObjectRequest) { - return AbortMultipartUploadRequest.builder() - .bucket(copyObjectRequest.destinationBucket()) - .key(copyObjectRequest.destinationKey()) - .requestPayer(copyObjectRequest.requestPayerAsString()) - .expectedBucketOwner(copyObjectRequest.expectedBucketOwner()); - } - - public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(PutObjectRequest putObjectRequest) { - return AbortMultipartUploadRequest.builder() - .bucket(putObjectRequest.bucket()) - .key(putObjectRequest.key()) - .requestPayer(putObjectRequest.requestPayerAsString()) - .expectedBucketOwner(putObjectRequest.expectedBucketOwner()); - } - - public static UploadPartCopyRequest toUploadPartCopyRequest(CopyObjectRequest copyObjectRequest, - int partNumber, - String uploadId, - String range) { - - return UploadPartCopyRequest.builder() - .sourceBucket(copyObjectRequest.sourceBucket()) - .sourceKey(copyObjectRequest.sourceKey()) - .sourceVersionId(copyObjectRequest.sourceVersionId()) - .uploadId(uploadId) - .partNumber(partNumber) - .destinationBucket(copyObjectRequest.destinationBucket()) - .destinationKey(copyObjectRequest.destinationKey()) - .copySourceIfMatch(copyObjectRequest.copySourceIfMatch()) - .copySourceIfNoneMatch(copyObjectRequest.copySourceIfNoneMatch()) - .copySourceIfUnmodifiedSince(copyObjectRequest.copySourceIfUnmodifiedSince()) - .copySourceRange(range) - .copySourceSSECustomerAlgorithm(copyObjectRequest.copySourceSSECustomerAlgorithm()) - .copySourceSSECustomerKeyMD5(copyObjectRequest.copySourceSSECustomerKeyMD5()) - .copySourceSSECustomerKey(copyObjectRequest.copySourceSSECustomerKey()) - .copySourceIfModifiedSince(copyObjectRequest.copySourceIfModifiedSince()) - .expectedBucketOwner(copyObjectRequest.expectedBucketOwner()) - .expectedSourceBucketOwner(copyObjectRequest.expectedSourceBucketOwner()) - .requestPayer(copyObjectRequest.requestPayerAsString()) - .sseCustomerKey(copyObjectRequest.sseCustomerKey()) - .sseCustomerAlgorithm(copyObjectRequest.sseCustomerAlgorithm()) - .sseCustomerKeyMD5(copyObjectRequest.sseCustomerKeyMD5()) - .build(); - } - - public static UploadPartRequest toUploadPartRequest(PutObjectRequest putObjectRequest, int partNumber, String uploadId) { - return UploadPartRequest.builder() - .bucket(putObjectRequest.bucket()) - .key(putObjectRequest.key()) - .uploadId(uploadId) - .partNumber(partNumber) - .sseCustomerAlgorithm(putObjectRequest.sseCustomerAlgorithm()) - .sseCustomerKeyMD5(putObjectRequest.sseCustomerKeyMD5()) - .sseCustomerKey(putObjectRequest.sseCustomerKey()) - .expectedBucketOwner(putObjectRequest.expectedBucketOwner()) - .requestPayer(putObjectRequest.requestPayerAsString()) - .sseCustomerKey(putObjectRequest.sseCustomerKey()) - .sseCustomerAlgorithm(putObjectRequest.sseCustomerAlgorithm()) - .sseCustomerKeyMD5(putObjectRequest.sseCustomerKeyMD5()) - .build(); - } - - public static PutObjectResponse toPutObjectResponse(CompleteMultipartUploadResponse response) { - PutObjectResponse.Builder builder = PutObjectResponse.builder() - .versionId(response.versionId()) - .checksumCRC32(response.checksumCRC32()) - .checksumSHA1(response.checksumSHA1()) - .checksumSHA256(response.checksumSHA256()) - .checksumCRC32C(response.checksumCRC32C()) - .eTag(response.eTag()) - .expiration(response.expiration()) - .bucketKeyEnabled(response.bucketKeyEnabled()) - .serverSideEncryption(response.serverSideEncryption()) - .ssekmsKeyId(response.ssekmsKeyId()) - .serverSideEncryption(response.serverSideEncryptionAsString()) - .requestCharged(response.requestChargedAsString()); - - // TODO: check why we have to do null check - if (response.responseMetadata() != null) { - builder.responseMetadata(response.responseMetadata()); - } - - if (response.sdkHttpResponse() != null) { - builder.sdkHttpResponse(response.sdkHttpResponse()); - } - - return builder.build(); - } -} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/UploadPartCopyRequestIterable.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/UploadPartCopyRequestIterable.java index f929bc3fc8f4..da8eea8fc64a 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/UploadPartCopyRequestIterable.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/UploadPartCopyRequestIterable.java @@ -19,6 +19,7 @@ import java.util.NoSuchElementException; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.pagination.sync.SdkIterable; +import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; @@ -65,7 +66,7 @@ public UploadPartCopyRequest next() { long partSize = Math.min(optimalPartSize, remainingBytes); String range = range(partSize); UploadPartCopyRequest uploadPartCopyRequest = - RequestConversionUtils.toUploadPartCopyRequest(copyObjectRequest, + SdkPojoConversionUtils.toUploadPartCopyRequest(copyObjectRequest, partNumber, uploadId, range); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index d043d88936c6..0f9be5070861 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -16,7 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; -import static software.amazon.awssdk.services.s3.internal.crt.RequestConversionUtils.toAbortMultipartUploadRequest; +import static software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils.toAbortMultipartUploadRequest; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -27,7 +27,6 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.internal.async.SplittingPublisher; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.internal.crt.RequestConversionUtils; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -60,8 +59,8 @@ public MultipartUploadHelper(S3AsyncClient s3AsyncClient, this.s3AsyncClient = s3AsyncClient; this.partSizeInBytes = partSizeInBytes; this.genericMultipartHelper = new GenericMultipartHelper<>(s3AsyncClient, - RequestConversionUtils::toAbortMultipartUploadRequest, - RequestConversionUtils::toPutObjectResponse); + SdkPojoConversionUtils::toAbortMultipartUploadRequest, + SdkPojoConversionUtils::toPutObjectResponse); this.maxMemoryUsageInBytes = maxMemoryUsageInBytes; this.multipartUploadThresholdInBytes = multipartUploadThresholdInBytes; } @@ -96,7 +95,7 @@ public CompletableFuture uploadObject(PutObjectRequest putObj private void uploadInParts(PutObjectRequest putObjectRequest, long contentLength, AsyncRequestBody asyncRequestBody, CompletableFuture returnFuture) { - CreateMultipartUploadRequest request = RequestConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); + CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -215,7 +214,7 @@ private void sendIndividualUploadPartRequest(String uploadId, private static CompletedPart convertUploadPartResponse(AtomicReferenceArray completedParts, Integer partNumber, UploadPartResponse uploadPartResponse) { - CompletedPart completedPart = RequestConversionUtils.toCompletedPart(uploadPartResponse, partNumber); + CompletedPart completedPart = SdkPojoConversionUtils.toCompletedPart(uploadPartResponse, partNumber); completedParts.set(partNumber - 1, completedPart); return completedPart; @@ -245,7 +244,7 @@ private static final class BodyToRequestConverter implements Function apply(AsyncRequestBody asyncRequestBody) { log.trace(() -> "Generating uploadPartRequest for partNumber " + partNumber); UploadPartRequest uploadRequest = - RequestConversionUtils.toUploadPartRequest(putObjectRequest, + SdkPojoConversionUtils.toUploadPartRequest(putObjectRequest, partNumber, uploadId); ++partNumber; diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtils.java new file mode 100644 index 000000000000..70512084150b --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtils.java @@ -0,0 +1,185 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.SdkPojo; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CopyObjectResult; +import software.amazon.awssdk.services.s3.model.CopyPartResult; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +/** + * Request conversion utility method for POJO classes associated with multipart feature. + */ +@SdkInternalApi +public final class SdkPojoConversionUtils { + + private static final HashSet PUT_OBJECT_REQUEST_TO_UPLOAD_PART_FIELDS_TO_IGNORE = + new HashSet<>(Arrays.asList("ChecksumSHA1", "ChecksumSHA256", "ContentMD5", "ChecksumCRC32C", "ChecksumCRC32")); + + private SdkPojoConversionUtils() { + } + + public static UploadPartRequest toUploadPartRequest(PutObjectRequest putObjectRequest, int partNumber, String uploadId) { + + UploadPartRequest.Builder builder = UploadPartRequest.builder(); + + setSdkFields(builder, putObjectRequest, PUT_OBJECT_REQUEST_TO_UPLOAD_PART_FIELDS_TO_IGNORE); + + return builder.uploadId(uploadId).partNumber(partNumber).build(); + } + + public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(PutObjectRequest putObjectRequest) { + + CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder(); + setSdkFields(builder, putObjectRequest); + return builder.build(); + } + + public static HeadObjectRequest toHeadObjectRequest(CopyObjectRequest copyObjectRequest) { + HeadObjectRequest.Builder builder = HeadObjectRequest.builder(); + setSdkFields(builder, copyObjectRequest); + return builder.build(); + } + + public static CompletedPart toCompletedPart(CopyPartResult copyPartResult, int partNumber) { + CompletedPart.Builder builder = CompletedPart.builder(); + + setSdkFields(builder, copyPartResult); + return builder.partNumber(partNumber).build(); + } + + public static CompletedPart toCompletedPart(UploadPartResponse partResponse, int partNumber) { + CompletedPart.Builder builder = CompletedPart.builder(); + setSdkFields(builder, partResponse); + return builder.partNumber(partNumber).build(); + } + + private static void setSdkFields(SdkPojo targetBuilder, SdkPojo sourceObject) { + setSdkFields(targetBuilder, sourceObject, new HashSet<>()); + } + + private static void setSdkFields(SdkPojo targetBuilder, SdkPojo sourceObject, Set fieldsToIgnore) { + Map sourceFields = retrieveSdkFields(sourceObject, sourceObject.sdkFields()); + List> targetSdkFields = targetBuilder.sdkFields(); + + for (SdkField field : targetSdkFields) { + if (fieldsToIgnore.contains(field.memberName())) { + continue; + } + field.set(targetBuilder, sourceFields.getOrDefault(field.memberName(), null)); + } + } + + public static CreateMultipartUploadRequest toCreateMultipartUploadRequest(CopyObjectRequest copyObjectRequest) { + CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder(); + + setSdkFields(builder, copyObjectRequest); + return builder.build(); + } + + public static CopyObjectResponse toCopyObjectResponse(CompleteMultipartUploadResponse response) { + CopyObjectResponse.Builder builder = CopyObjectResponse.builder(); + + setSdkFields(builder, response); + + if (response.responseMetadata() != null) { + builder.responseMetadata(response.responseMetadata()); + } + + if (response.sdkHttpResponse() != null) { + builder.sdkHttpResponse(response.sdkHttpResponse()); + } + + return builder.copyObjectResult(toCopyObjectResult(response)) + .build(); + } + + private static CopyObjectResult toCopyObjectResult(CompleteMultipartUploadResponse response) { + CopyObjectResult.Builder builder = CopyObjectResult.builder(); + + setSdkFields(builder, response); + return builder.build(); + } + + public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(CopyObjectRequest copyObjectRequest) { + AbortMultipartUploadRequest.Builder builder = AbortMultipartUploadRequest.builder(); + setSdkFields(builder, copyObjectRequest); + return builder; + } + + public static AbortMultipartUploadRequest.Builder toAbortMultipartUploadRequest(PutObjectRequest putObjectRequest) { + AbortMultipartUploadRequest.Builder builder = AbortMultipartUploadRequest.builder(); + setSdkFields(builder, putObjectRequest); + return builder; + } + + public static UploadPartCopyRequest toUploadPartCopyRequest(CopyObjectRequest copyObjectRequest, + int partNumber, + String uploadId, + String range) { + UploadPartCopyRequest.Builder builder = UploadPartCopyRequest.builder(); + setSdkFields(builder, copyObjectRequest); + return builder.copySourceRange(range) + .partNumber(partNumber) + .uploadId(uploadId) + .build(); + } + + public static PutObjectResponse toPutObjectResponse(CompleteMultipartUploadResponse response) { + + PutObjectResponse.Builder builder = PutObjectResponse.builder(); + + setSdkFields(builder, response); + + // TODO: check why we have to do null check + if (response.responseMetadata() != null) { + builder.responseMetadata(response.responseMetadata()); + } + + if (response.sdkHttpResponse() != null) { + builder.sdkHttpResponse(response.sdkHttpResponse()); + } + + return builder.build(); + } + + private static Map retrieveSdkFields(SdkPojo sourceObject, List> sdkFields) { + return sdkFields.stream().collect( + HashMap::new, + (map, field) -> map.put(field.memberName(), + field.getValueOrDefault(sourceObject)), + Map::putAll); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelperTest.java index 0db53c246e03..1ea17d4ba967 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelperTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelperTest.java @@ -17,6 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -27,6 +28,9 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -164,7 +168,12 @@ void mpu_onePartFailed_shouldFailOtherPartsAndAbort() { AbortMultipartUploadRequest actualRequest = argumentCaptor.getValue(); assertThat(actualRequest.uploadId()).isEqualTo(UPLOAD_ID); - assertThat(ongoingRequest).isCompletedExceptionally(); + try { + ongoingRequest.get(1, TimeUnit.MILLISECONDS); + fail("no exception thrown"); + } catch (Exception e) { + assertThat(e.getCause()).hasMessageContaining("request failed"); + } } @Test diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/CopyRequestConversionUtilsTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtilsTest.java similarity index 63% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/CopyRequestConversionUtilsTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtilsTest.java index 104d5f6e045f..4d5a333a51dd 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/CopyRequestConversionUtilsTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/SdkPojoConversionUtilsTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.internal.crt; +package software.amazon.awssdk.services.s3.internal.multipart; import static org.assertj.core.api.Assertions.assertThat; @@ -35,6 +35,7 @@ import software.amazon.awssdk.core.SdkField; import software.amazon.awssdk.core.SdkPojo; import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedPart; @@ -43,19 +44,23 @@ import software.amazon.awssdk.services.s3.model.CopyPartResult; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3ResponseMetadata; import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.utils.Logger; -class CopyRequestConversionUtilsTest { - private static final Logger log = Logger.loggerFor(RequestConversionUtils.class); +class SdkPojoConversionUtilsTest { + private static final Logger log = Logger.loggerFor(SdkPojoConversionUtils.class); private static final Random RNG = new Random(); @Test void toHeadObject_shouldCopyProperties() { CopyObjectRequest randomCopyObject = randomCopyObjectRequest(); - HeadObjectRequest convertedToHeadObject = RequestConversionUtils.toHeadObjectRequest(randomCopyObject); + HeadObjectRequest convertedToHeadObject = SdkPojoConversionUtils.toHeadObjectRequest(randomCopyObject); Set fieldsToIgnore = new HashSet<>(Arrays.asList("ExpectedBucketOwner", "RequestPayer", "Bucket", @@ -69,12 +74,12 @@ void toHeadObject_shouldCopyProperties() { } @Test - void toCompletedPart_shouldCopyProperties() { + void toCompletedPart_copy_shouldCopyProperties() { CopyPartResult.Builder fromObject = CopyPartResult.builder(); setFieldsToRandomValues(fromObject.sdkFields(), fromObject); CopyPartResult result = fromObject.build(); - CompletedPart convertedCompletedPart = RequestConversionUtils.toCompletedPart(result, 1); + CompletedPart convertedCompletedPart = SdkPojoConversionUtils.toCompletedPart(result, 1); verifyFieldsAreCopied(result, convertedCompletedPart, new HashSet<>(), CopyPartResult.builder().sdkFields(), CompletedPart.builder().sdkFields()); @@ -82,9 +87,9 @@ void toCompletedPart_shouldCopyProperties() { } @Test - void toCreateMultipartUploadRequest_shouldCopyProperties() { + void toCreateMultipartUploadRequest_copyObject_shouldCopyProperties() { CopyObjectRequest randomCopyObject = randomCopyObjectRequest(); - CreateMultipartUploadRequest convertedRequest = RequestConversionUtils.toCreateMultipartUploadRequest(randomCopyObject); + CreateMultipartUploadRequest convertedRequest = SdkPojoConversionUtils.toCreateMultipartUploadRequest(randomCopyObject); Set fieldsToIgnore = new HashSet<>(); verifyFieldsAreCopied(randomCopyObject, convertedRequest, fieldsToIgnore, CopyObjectRequest.builder().sdkFields(), @@ -100,7 +105,7 @@ void toCopyObjectResponse_shouldCopyProperties() { responseBuilder.responseMetadata(s3ResponseMetadata).sdkHttpResponse(sdkHttpFullResponse); CompleteMultipartUploadResponse result = responseBuilder.build(); - CopyObjectResponse convertedRequest = RequestConversionUtils.toCopyObjectResponse(result); + CopyObjectResponse convertedRequest = SdkPojoConversionUtils.toCopyObjectResponse(result); Set fieldsToIgnore = new HashSet<>(); verifyFieldsAreCopied(result, convertedRequest, fieldsToIgnore, CompleteMultipartUploadResponse.builder().sdkFields(), @@ -111,21 +116,29 @@ void toCopyObjectResponse_shouldCopyProperties() { } @Test - void toAbortMultipartUploadRequest_shouldCopyProperties() { + void toAbortMultipartUploadRequest_copyObject_shouldCopyProperties() { CopyObjectRequest randomCopyObject = randomCopyObjectRequest(); - AbortMultipartUploadRequest convertedRequest = RequestConversionUtils.toAbortMultipartUploadRequest(randomCopyObject).build(); + AbortMultipartUploadRequest convertedRequest = SdkPojoConversionUtils.toAbortMultipartUploadRequest(randomCopyObject).build(); Set fieldsToIgnore = new HashSet<>(); verifyFieldsAreCopied(randomCopyObject, convertedRequest, fieldsToIgnore, CopyObjectRequest.builder().sdkFields(), AbortMultipartUploadRequest.builder().sdkFields()); + } - //assertThat(convertedRequest.uploadId()).isEqualTo("id"); + @Test + void toAbortMultipartUploadRequest_putObject_shouldCopyProperties() { + PutObjectRequest randomCopyObject = randomPutObjectRequest(); + AbortMultipartUploadRequest convertedRequest = SdkPojoConversionUtils.toAbortMultipartUploadRequest(randomCopyObject).build(); + Set fieldsToIgnore = new HashSet<>(); + verifyFieldsAreCopied(randomCopyObject, convertedRequest, fieldsToIgnore, + PutObjectRequest.builder().sdkFields(), + AbortMultipartUploadRequest.builder().sdkFields()); } @Test void toUploadPartCopyRequest_shouldCopyProperties() { CopyObjectRequest randomCopyObject = randomCopyObjectRequest(); - UploadPartCopyRequest convertedObject = RequestConversionUtils.toUploadPartCopyRequest(randomCopyObject, 1, "id", + UploadPartCopyRequest convertedObject = SdkPojoConversionUtils.toUploadPartCopyRequest(randomCopyObject, 1, "id", "bytes=0-1024"); Set fieldsToIgnore = new HashSet<>(Collections.singletonList("CopySource")); verifyFieldsAreCopied(randomCopyObject, convertedObject, fieldsToIgnore, @@ -133,6 +146,61 @@ void toUploadPartCopyRequest_shouldCopyProperties() { UploadPartCopyRequest.builder().sdkFields()); } + @Test + void toUploadPartRequest_shouldCopyProperties() { + PutObjectRequest randomObject = randomPutObjectRequest(); + UploadPartRequest convertedObject = SdkPojoConversionUtils.toUploadPartRequest(randomObject, 1, "id"); + Set fieldsToIgnore = new HashSet<>(Arrays.asList("ChecksumCRC32", "ChecksumSHA256", "ContentMD5", "ChecksumSHA1", + "ChecksumCRC32C")); + verifyFieldsAreCopied(randomObject, convertedObject, fieldsToIgnore, + PutObjectRequest.builder().sdkFields(), + UploadPartRequest.builder().sdkFields()); + assertThat(convertedObject.partNumber()).isEqualTo(1); + assertThat(convertedObject.uploadId()).isEqualTo("id"); + } + + @Test + void toPutObjectResponse_shouldCopyProperties() { + CompleteMultipartUploadResponse.Builder builder = CompleteMultipartUploadResponse.builder(); + populateFields(builder); + S3ResponseMetadata s3ResponseMetadata = S3ResponseMetadata.create(DefaultAwsResponseMetadata.create(new HashMap<>())); + SdkHttpFullResponse sdkHttpFullResponse = SdkHttpFullResponse.builder().statusCode(200).build(); + builder.responseMetadata(s3ResponseMetadata).sdkHttpResponse(sdkHttpFullResponse); + CompleteMultipartUploadResponse randomObject = builder.build(); + PutObjectResponse convertedObject = SdkPojoConversionUtils.toPutObjectResponse(randomObject); + Set fieldsToIgnore = new HashSet<>(); + verifyFieldsAreCopied(randomObject, convertedObject, fieldsToIgnore, + CompleteMultipartUploadResponse.builder().sdkFields(), + PutObjectResponse.builder().sdkFields()); + + assertThat(convertedObject.sdkHttpResponse()).isEqualTo(sdkHttpFullResponse); + assertThat(convertedObject.responseMetadata()).isEqualTo(s3ResponseMetadata); + } + + @Test + void toCreateMultipartUploadRequest_putObjectRequest_shouldCopyProperties() { + PutObjectRequest randomObject = randomPutObjectRequest(); + CreateMultipartUploadRequest convertedObject = SdkPojoConversionUtils.toCreateMultipartUploadRequest(randomObject); + Set fieldsToIgnore = new HashSet<>(); + System.out.println(convertedObject); + verifyFieldsAreCopied(randomObject, convertedObject, fieldsToIgnore, + PutObjectRequest.builder().sdkFields(), + CreateMultipartUploadRequest.builder().sdkFields()); + } + + @Test + void toCompletedPart_putObject_shouldCopyProperties() { + UploadPartResponse.Builder fromObject = UploadPartResponse.builder(); + setFieldsToRandomValues(fromObject.sdkFields(), fromObject); + UploadPartResponse result = fromObject.build(); + + CompletedPart convertedCompletedPart = SdkPojoConversionUtils.toCompletedPart(result, 1); + verifyFieldsAreCopied(result, convertedCompletedPart, new HashSet<>(), + UploadPartResponse.builder().sdkFields(), + CompletedPart.builder().sdkFields()); + assertThat(convertedCompletedPart.partNumber()).isEqualTo(1); + } + private static void verifyFieldsAreCopied(SdkPojo requestConvertedFrom, SdkPojo requestConvertedTo, Set fieldsToIgnore, @@ -147,7 +215,7 @@ private static void verifyFieldsAreCopied(SdkPojo requestConvertedFrom, SdkField toField = toObjectEntry.getValue(); if (fieldsToIgnore.contains(toField.memberName())) { - log.info(() -> "Ignoring fields: " + toField.locationName()); + log.info(() -> "Ignoring fields: " + toField.memberName()); continue; } @@ -155,7 +223,7 @@ private static void verifyFieldsAreCopied(SdkPojo requestConvertedFrom, if (fromField == null) { log.info(() -> String.format("Ignoring field [%s] because the object to convert from does not have such field ", - toField.locationName())); + toField.memberName())); continue; } @@ -175,6 +243,16 @@ private CopyObjectRequest randomCopyObjectRequest() { return builder.build(); } + private PutObjectRequest randomPutObjectRequest() { + PutObjectRequest.Builder builder = PutObjectRequest.builder(); + setFieldsToRandomValues(builder.sdkFields(), builder); + return builder.build(); + } + + private void populateFields(SdkPojo pojo) { + setFieldsToRandomValues(pojo.sdkFields(), pojo); + } + private void setFieldsToRandomValues(Collection> fields, Object builder) { for (SdkField f : fields) { setFieldToRandomValue(f, builder); @@ -193,6 +271,8 @@ private static void setFieldToRandomValue(SdkField sdkField, Object obj) { sdkField.set(obj, new HashMap<>()); } else if (targetClass.equals(Boolean.class)) { sdkField.set(obj, true); + } else if (targetClass.equals(Long.class)) { + sdkField.set(obj, randomLong()); } else { throw new IllegalArgumentException("Unknown SdkField type: " + targetClass + " name: " + sdkField.memberName()); } @@ -201,7 +281,7 @@ private static void setFieldToRandomValue(SdkField sdkField, Object obj) { private static Map> sdkFieldMap(Collection> sdkFields) { Map> map = new HashMap<>(sdkFields.size()); for (SdkField f : sdkFields) { - String locName = f.locationName(); + String locName = f.memberName(); if (map.put(locName, f) != null) { throw new IllegalArgumentException("Multiple SdkFields map to same location name"); } @@ -216,4 +296,8 @@ private static Instant randomInstant() { private static Integer randomInteger() { return RNG.nextInt(); } + + private static long randomLong() { + return RNG.nextLong(); + } }