Skip to content

Commit 03cd6fb

Browse files
steve-todorovsteve-todorov
authored
Upgrade AWS SDK to latest version (#860)
* deps(upgrade): Upgrade software.amazon.awssdk:bom from 2.16.28 to 2.17.295 * deps(upgrade): Upgrade software.amazon.awssdk:bom from 2.17.295 to 2.18.0 * deps(upgrade): Upgrade software.amazon.awssdk:bom from 2.18.0 to 2.23.7 * deps(upgrade): Upgrade software.amazon.awssdk:bom from 2.23.7 to 2.25.70 * deps(upgrade): Upgrade software.amazon.awssdk:bom from 2.25.70 to 2.29.9
1 parent 4946d83 commit 03cd6fb

File tree

4 files changed

+255
-41
lines changed

4 files changed

+255
-41
lines changed

‎build.gradle.kts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,7 @@ java {
3939
}
4040

4141
dependencies {
42-
api(platform("software.amazon.awssdk:bom:2.16.28")) {
43-
// Change the `AWS_VERSION` in ./docs/mkocs.yaml file.
44-
}
42+
api(platform("software.amazon.awssdk:bom:2.29.9"))
4543
api("software.amazon.awssdk:s3") {
4644
exclude("commons-logging", "commons-logging")
4745
}

‎src/test/java/org/carlspring/cloud/storage/s3fs/S3ClientFactoryTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ void theDefaults()
127127
assertNull(proxyConfiguration.password());
128128
assertNull(proxyConfiguration.ntlmDomain());
129129
assertNull(proxyConfiguration.ntlmWorkstation());
130-
assertNull(proxyConfiguration.scheme());
130+
assertEquals(proxyConfiguration.scheme(), "http"); // default is now "http" instead of null.
131131

132132
S3Configuration serviceConfiguration = clientFactory.getServiceConfiguration(props);
133133
assertFalse(serviceConfiguration.pathStyleAccessEnabled());

‎src/test/java/org/carlspring/cloud/storage/s3fs/util/ExposingS3BaseClientBuilder.java

Lines changed: 241 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,282 @@
11
package org.carlspring.cloud.storage.s3fs.util;
22

3-
import java.util.List;
4-
5-
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
63
import software.amazon.awssdk.awscore.client.builder.AwsDefaultClientBuilder;
7-
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
4+
import software.amazon.awssdk.awscore.client.config.AwsClientOption;
5+
import software.amazon.awssdk.core.SdkPlugin;
86
import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
97
import software.amazon.awssdk.core.client.config.SdkClientOption;
108
import software.amazon.awssdk.core.interceptor.ClasspathInterceptorChainFactory;
119
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
12-
import software.amazon.awssdk.core.signer.Signer;
10+
import software.amazon.awssdk.http.auth.aws.scheme.AwsV4AuthScheme;
11+
import software.amazon.awssdk.http.auth.aws.scheme.AwsV4aAuthScheme;
12+
import software.amazon.awssdk.http.auth.scheme.NoAuthAuthScheme;
13+
import software.amazon.awssdk.http.auth.spi.scheme.AuthScheme;
14+
import software.amazon.awssdk.identity.spi.IdentityProvider;
15+
import software.amazon.awssdk.identity.spi.IdentityProviders;
1316
import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
1417
import software.amazon.awssdk.services.s3.S3Configuration;
15-
import software.amazon.awssdk.services.s3.S3Configuration.Builder;
18+
import software.amazon.awssdk.services.s3.auth.scheme.S3AuthSchemeProvider;
19+
import software.amazon.awssdk.services.s3.auth.scheme.internal.S3AuthSchemeInterceptor;
20+
import software.amazon.awssdk.services.s3.endpoints.S3ClientContextParams;
21+
import software.amazon.awssdk.services.s3.endpoints.S3EndpointProvider;
22+
import software.amazon.awssdk.services.s3.endpoints.internal.S3RequestSetEndpointInterceptor;
23+
import software.amazon.awssdk.services.s3.endpoints.internal.S3ResolveEndpointInterceptor;
24+
import software.amazon.awssdk.services.s3.internal.S3ServiceClientConfigurationBuilder;
25+
import software.amazon.awssdk.services.s3.internal.endpoints.UseGlobalEndpointResolver;
26+
import software.amazon.awssdk.services.s3.internal.handlers.AsyncChecksumValidationInterceptor;
27+
import software.amazon.awssdk.services.s3.internal.handlers.CreateBucketInterceptor;
28+
import software.amazon.awssdk.services.s3.internal.handlers.CreateMultipartUploadRequestInterceptor;
29+
import software.amazon.awssdk.services.s3.internal.handlers.DecodeUrlEncodedResponseInterceptor;
30+
import software.amazon.awssdk.services.s3.internal.handlers.EnableTrailingChecksumInterceptor;
31+
import software.amazon.awssdk.services.s3.internal.handlers.ExceptionTranslationInterceptor;
32+
import software.amazon.awssdk.services.s3.internal.handlers.GetBucketPolicyInterceptor;
33+
import software.amazon.awssdk.services.s3.internal.handlers.GetObjectInterceptor;
34+
import software.amazon.awssdk.services.s3.internal.handlers.ObjectMetadataInterceptor;
35+
import software.amazon.awssdk.services.s3.internal.handlers.S3ExpressChecksumInterceptor;
36+
import software.amazon.awssdk.services.s3.internal.handlers.StreamingRequestInterceptor;
37+
import software.amazon.awssdk.services.s3.internal.handlers.SyncChecksumValidationInterceptor;
38+
import software.amazon.awssdk.services.s3.internal.plugins.S3DisableChunkEncodingIfConfiguredPlugin;
39+
import software.amazon.awssdk.services.s3.internal.s3express.S3ExpressPlugin;
40+
import software.amazon.awssdk.services.s3.internal.s3express.UseS3ExpressAuthResolver;
1641
import software.amazon.awssdk.utils.CollectionUtils;
42+
import software.amazon.awssdk.utils.Validate;
1743

44+
import java.util.ArrayList;
45+
import java.util.HashMap;
46+
import java.util.List;
47+
import java.util.Map;
48+
49+
/**
50+
* This class follows the {@link software.amazon.awssdk.services.s3.DefaultS3BaseClientBuilder} class which is not public so we can't extend it no expose it.
51+
*
52+
* @param <B>
53+
* @param <C>
54+
*/
1855
abstract class ExposingS3BaseClientBuilder<B extends S3BaseClientBuilder<B, C>, C>
1956
extends AwsDefaultClientBuilder<B, C>
2057
{
2158

59+
private final Map<String, AuthScheme<?>> additionalAuthSchemes = new HashMap();
60+
2261
ExposingS3BaseClientBuilder()
2362
{
2463
}
2564

65+
@Override
2666
protected final String serviceEndpointPrefix()
2767
{
2868
return "s3";
2969
}
3070

71+
@Override
3172
protected final String serviceName()
3273
{
3374
return "S3";
3475
}
3576

36-
protected final SdkClientConfiguration mergeServiceDefaults(SdkClientConfiguration config)
37-
{
38-
return config.merge((c) -> c.option(SdkAdvancedClientOption.SIGNER, this.defaultSigner()).option(
39-
SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false).option(
40-
SdkClientOption.SERVICE_CONFIGURATION, S3Configuration.builder().build()));
77+
@Override
78+
protected final SdkClientConfiguration mergeServiceDefaults(SdkClientConfiguration config) {
79+
return config.merge(c -> c.option(SdkClientOption.ENDPOINT_PROVIDER, defaultEndpointProvider())
80+
.option(SdkClientOption.AUTH_SCHEME_PROVIDER, defaultAuthSchemeProvider())
81+
.option(SdkClientOption.AUTH_SCHEMES, authSchemes())
82+
.option(SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false)
83+
.option(SdkClientOption.SERVICE_CONFIGURATION, S3Configuration.builder().build()));
4184
}
4285

43-
protected final SdkClientConfiguration finalizeServiceConfiguration(SdkClientConfiguration config)
44-
{
86+
@Override
87+
protected final SdkClientConfiguration finalizeServiceConfiguration(SdkClientConfiguration config) {
88+
List<ExecutionInterceptor> endpointInterceptors = new ArrayList<>();
89+
endpointInterceptors.add(new S3AuthSchemeInterceptor());
90+
endpointInterceptors.add(new S3ResolveEndpointInterceptor());
91+
endpointInterceptors.add(new S3RequestSetEndpointInterceptor());
92+
endpointInterceptors.add(new StreamingRequestInterceptor());
93+
endpointInterceptors.add(new CreateBucketInterceptor());
94+
endpointInterceptors.add(new CreateMultipartUploadRequestInterceptor());
95+
endpointInterceptors.add(new DecodeUrlEncodedResponseInterceptor());
96+
endpointInterceptors.add(new GetBucketPolicyInterceptor());
97+
endpointInterceptors.add(new S3ExpressChecksumInterceptor());
98+
endpointInterceptors.add(new AsyncChecksumValidationInterceptor());
99+
endpointInterceptors.add(new SyncChecksumValidationInterceptor());
100+
endpointInterceptors.add(new EnableTrailingChecksumInterceptor());
101+
endpointInterceptors.add(new ExceptionTranslationInterceptor());
102+
endpointInterceptors.add(new GetObjectInterceptor());
103+
endpointInterceptors.add(new ObjectMetadataInterceptor());
45104
ClasspathInterceptorChainFactory interceptorFactory = new ClasspathInterceptorChainFactory();
46-
List<ExecutionInterceptor> interceptors = interceptorFactory.getInterceptors(
47-
"software/amazon/awssdk/services/s3/execution.interceptors");
48-
interceptors = CollectionUtils.mergeLists(interceptors,
49-
config.option(SdkClientOption.EXECUTION_INTERCEPTORS));
50-
Builder c = ((S3Configuration) config.option(SdkClientOption.SERVICE_CONFIGURATION)).toBuilder();
51-
c.profileFile(c.profileFile() != null ? c.profileFile() :
52-
config.option(SdkClientOption.PROFILE_FILE)).profileName(
53-
c.profileName() != null ? c.profileName() : config.option(SdkClientOption.PROFILE_NAME));
54-
return config.toBuilder().option(SdkClientOption.EXECUTION_INTERCEPTORS, interceptors).option(
55-
SdkClientOption.SERVICE_CONFIGURATION, c.build()).build();
56-
}
57-
58-
private Signer defaultSigner()
59-
{
60-
return AwsS3V4Signer.create();
105+
List<ExecutionInterceptor> interceptors = interceptorFactory
106+
.getInterceptors("software/amazon/awssdk/services/s3/execution.interceptors");
107+
List<ExecutionInterceptor> additionalInterceptors = new ArrayList<>();
108+
interceptors = CollectionUtils.mergeLists(endpointInterceptors, interceptors);
109+
interceptors = CollectionUtils.mergeLists(interceptors, additionalInterceptors);
110+
interceptors = CollectionUtils.mergeLists(interceptors, config.option(SdkClientOption.EXECUTION_INTERCEPTORS));
111+
S3Configuration.Builder serviceConfigBuilder = ((S3Configuration) config.option(SdkClientOption.SERVICE_CONFIGURATION))
112+
.toBuilder();
113+
serviceConfigBuilder.profileFile(serviceConfigBuilder.profileFileSupplier() != null ? serviceConfigBuilder
114+
.profileFileSupplier() : config.option(SdkClientOption.PROFILE_FILE_SUPPLIER));
115+
serviceConfigBuilder.profileName(serviceConfigBuilder.profileName() != null ? serviceConfigBuilder.profileName() : config
116+
.option(SdkClientOption.PROFILE_NAME));
117+
if (serviceConfigBuilder.dualstackEnabled() != null) {
118+
Validate.validState(
119+
config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED) == null,
120+
"Dualstack has been configured on both S3Configuration and the client/global level. Please limit dualstack configuration to one location.");
121+
} else {
122+
serviceConfigBuilder.dualstackEnabled(config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED));
123+
}
124+
if (serviceConfigBuilder.useArnRegionEnabled() != null) {
125+
Validate.validState(
126+
clientContextParams.get(S3ClientContextParams.USE_ARN_REGION) == null,
127+
"UseArnRegion has been configured on both S3Configuration and the client/global level. Please limit UseArnRegion configuration to one location.");
128+
} else {
129+
serviceConfigBuilder.useArnRegionEnabled(clientContextParams.get(S3ClientContextParams.USE_ARN_REGION));
130+
}
131+
if (serviceConfigBuilder.multiRegionEnabled() != null) {
132+
Validate.validState(
133+
clientContextParams.get(S3ClientContextParams.DISABLE_MULTI_REGION_ACCESS_POINTS) == null,
134+
"DisableMultiRegionAccessPoints has been configured on both S3Configuration and the client/global level. Please limit DisableMultiRegionAccessPoints configuration to one location.");
135+
} else if (clientContextParams.get(S3ClientContextParams.DISABLE_MULTI_REGION_ACCESS_POINTS) != null) {
136+
serviceConfigBuilder.multiRegionEnabled(!clientContextParams
137+
.get(S3ClientContextParams.DISABLE_MULTI_REGION_ACCESS_POINTS));
138+
}
139+
if (serviceConfigBuilder.pathStyleAccessEnabled() != null) {
140+
Validate.validState(
141+
clientContextParams.get(S3ClientContextParams.FORCE_PATH_STYLE) == null,
142+
"ForcePathStyle has been configured on both S3Configuration and the client/global level. Please limit ForcePathStyle configuration to one location.");
143+
} else {
144+
serviceConfigBuilder.pathStyleAccessEnabled(clientContextParams.get(S3ClientContextParams.FORCE_PATH_STYLE));
145+
}
146+
if (serviceConfigBuilder.accelerateModeEnabled() != null) {
147+
Validate.validState(
148+
clientContextParams.get(S3ClientContextParams.ACCELERATE) == null,
149+
"Accelerate has been configured on both S3Configuration and the client/global level. Please limit Accelerate configuration to one location.");
150+
} else {
151+
serviceConfigBuilder.accelerateModeEnabled(clientContextParams.get(S3ClientContextParams.ACCELERATE));
152+
}
153+
S3Configuration finalServiceConfig = serviceConfigBuilder.build();
154+
clientContextParams.put(S3ClientContextParams.USE_ARN_REGION, finalServiceConfig.useArnRegionEnabled());
155+
clientContextParams.put(S3ClientContextParams.DISABLE_MULTI_REGION_ACCESS_POINTS,
156+
!finalServiceConfig.multiRegionEnabled());
157+
clientContextParams.put(S3ClientContextParams.FORCE_PATH_STYLE, finalServiceConfig.pathStyleAccessEnabled());
158+
clientContextParams.put(S3ClientContextParams.ACCELERATE, finalServiceConfig.accelerateModeEnabled());
159+
UseGlobalEndpointResolver globalEndpointResolver = new UseGlobalEndpointResolver(config);
160+
UseS3ExpressAuthResolver useS3ExpressAuthResolver = new UseS3ExpressAuthResolver(config);
161+
if (clientContextParams.get(S3ClientContextParams.DISABLE_S3_EXPRESS_SESSION_AUTH) == null) {
162+
clientContextParams.put(S3ClientContextParams.DISABLE_S3_EXPRESS_SESSION_AUTH, !useS3ExpressAuthResolver.resolve());
163+
}
164+
SdkClientConfiguration.Builder builder = config.toBuilder();
165+
builder.lazyOption(SdkClientOption.IDENTITY_PROVIDERS, c -> {
166+
IdentityProviders.Builder result = IdentityProviders.builder();
167+
IdentityProvider<?> credentialsIdentityProvider = c.get(AwsClientOption.CREDENTIALS_IDENTITY_PROVIDER);
168+
if (credentialsIdentityProvider != null) {
169+
result.putIdentityProvider(credentialsIdentityProvider);
170+
}
171+
return result.build();
172+
});
173+
builder.option(SdkClientOption.EXECUTION_INTERCEPTORS, interceptors)
174+
.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED, finalServiceConfig.dualstackEnabled())
175+
.option(SdkClientOption.SERVICE_CONFIGURATION, finalServiceConfig)
176+
.option(AwsClientOption.USE_GLOBAL_ENDPOINT,
177+
globalEndpointResolver.resolve(config.option(AwsClientOption.AWS_REGION)))
178+
.option(SdkClientOption.CLIENT_CONTEXT_PARAMS, clientContextParams.build());
179+
return builder.build();
61180
}
62181

63-
protected final String signingName()
64-
{
182+
@Override
183+
protected final String signingName() {
65184
return "s3";
66185
}
67186

68-
public B serviceConfiguration(S3Configuration serviceConfiguration)
69-
{
70-
this.clientConfiguration.option(SdkClientOption.SERVICE_CONFIGURATION, serviceConfiguration);
71-
return this.thisBuilder();
187+
private S3EndpointProvider defaultEndpointProvider() {
188+
return S3EndpointProvider.defaultProvider();
72189
}
73190

74-
public void setServiceConfiguration(S3Configuration serviceConfiguration)
75-
{
76-
this.serviceConfiguration(serviceConfiguration);
191+
public B authSchemeProvider(S3AuthSchemeProvider authSchemeProvider) {
192+
clientConfiguration.option(SdkClientOption.AUTH_SCHEME_PROVIDER, authSchemeProvider);
193+
return thisBuilder();
194+
}
195+
196+
private S3AuthSchemeProvider defaultAuthSchemeProvider() {
197+
return S3AuthSchemeProvider.defaultProvider();
198+
}
199+
200+
@Override
201+
public B putAuthScheme(AuthScheme<?> authScheme) {
202+
additionalAuthSchemes.put(authScheme.schemeId(), authScheme);
203+
return thisBuilder();
204+
}
205+
206+
private Map<String, AuthScheme<?>> authSchemes() {
207+
Map<String, AuthScheme<?>> schemes = new HashMap<>(3 + this.additionalAuthSchemes.size());
208+
AwsV4AuthScheme awsV4AuthScheme = AwsV4AuthScheme.create();
209+
schemes.put(awsV4AuthScheme.schemeId(), awsV4AuthScheme);
210+
AwsV4aAuthScheme awsV4aAuthScheme = AwsV4aAuthScheme.create();
211+
schemes.put(awsV4aAuthScheme.schemeId(), awsV4aAuthScheme);
212+
NoAuthAuthScheme noAuthAuthScheme = NoAuthAuthScheme.create();
213+
schemes.put(noAuthAuthScheme.schemeId(), noAuthAuthScheme);
214+
schemes.putAll(this.additionalAuthSchemes);
215+
return schemes;
216+
}
217+
218+
public B accelerate(Boolean accelerate) {
219+
clientContextParams.put(S3ClientContextParams.ACCELERATE, accelerate);
220+
return thisBuilder();
221+
}
222+
223+
public B disableMultiRegionAccessPoints(Boolean disableMultiRegionAccessPoints) {
224+
clientContextParams.put(S3ClientContextParams.DISABLE_MULTI_REGION_ACCESS_POINTS, disableMultiRegionAccessPoints);
225+
return thisBuilder();
226+
}
227+
228+
public B disableS3ExpressSessionAuth(Boolean disableS3ExpressSessionAuth) {
229+
clientContextParams.put(S3ClientContextParams.DISABLE_S3_EXPRESS_SESSION_AUTH, disableS3ExpressSessionAuth);
230+
return thisBuilder();
231+
}
232+
233+
public B forcePathStyle(Boolean forcePathStyle) {
234+
clientContextParams.put(S3ClientContextParams.FORCE_PATH_STYLE, forcePathStyle);
235+
return thisBuilder();
236+
}
237+
238+
public B useArnRegion(Boolean useArnRegion) {
239+
clientContextParams.put(S3ClientContextParams.USE_ARN_REGION, useArnRegion);
240+
return thisBuilder();
241+
}
242+
243+
public B crossRegionAccessEnabled(Boolean crossRegionAccessEnabled) {
244+
clientContextParams.put(S3ClientContextParams.CROSS_REGION_ACCESS_ENABLED, crossRegionAccessEnabled);
245+
return thisBuilder();
246+
}
247+
248+
public B serviceConfiguration(S3Configuration serviceConfiguration) {
249+
clientConfiguration.option(SdkClientOption.SERVICE_CONFIGURATION, serviceConfiguration);
250+
return thisBuilder();
251+
}
252+
253+
public void setServiceConfiguration(S3Configuration serviceConfiguration) {
254+
serviceConfiguration(serviceConfiguration);
255+
}
256+
257+
@Override
258+
protected SdkClientConfiguration invokePlugins(SdkClientConfiguration config) {
259+
List<SdkPlugin> internalPlugins = internalPlugins(config);
260+
List<SdkPlugin> externalPlugins = plugins();
261+
if (internalPlugins.isEmpty() && externalPlugins.isEmpty()) {
262+
return config;
263+
}
264+
List<SdkPlugin> plugins = CollectionUtils.mergeLists(internalPlugins, externalPlugins);
265+
SdkClientConfiguration.Builder configuration = config.toBuilder();
266+
S3ServiceClientConfigurationBuilder serviceConfigBuilder = new S3ServiceClientConfigurationBuilder(configuration);
267+
for (SdkPlugin plugin : plugins) {
268+
plugin.configureClient(serviceConfigBuilder);
269+
}
270+
return configuration.build();
271+
}
272+
273+
private List<SdkPlugin> internalPlugins(SdkClientConfiguration config) {
274+
List<SdkPlugin> internalPlugins = new ArrayList<>();
275+
internalPlugins.add(new S3DisableChunkEncodingIfConfiguredPlugin((config)));
276+
internalPlugins.add(new S3ExpressPlugin());
277+
return internalPlugins;
278+
}
279+
280+
protected static void validateClientOptions(SdkClientConfiguration c) {
77281
}
78282
}

‎src/test/java/org/carlspring/cloud/storage/s3fs/util/ExposingS3ClientBuilder.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
package org.carlspring.cloud.storage.s3fs.util;
22

3+
import software.amazon.awssdk.core.client.config.SdkClientOption;
34
import software.amazon.awssdk.services.s3.S3Client;
45
import software.amazon.awssdk.services.s3.S3ClientBuilder;
6+
import software.amazon.awssdk.services.s3.endpoints.S3EndpointProvider;
57

8+
/**
9+
* This class follows the {@link software.amazon.awssdk.services.s3.DefaultS3ClientBuilder} implementation which is not public.
10+
*/
611
public class ExposingS3ClientBuilder
712
extends ExposingS3BaseClientBuilder<S3ClientBuilder, S3Client>
813
implements S3ClientBuilder
@@ -16,4 +21,11 @@ protected final S3Client buildClient()
1621
{
1722
return new ExposingS3Client(super.syncClientConfiguration());
1823
}
24+
25+
26+
public ExposingS3ClientBuilder endpointProvider(S3EndpointProvider endpointProvider) {
27+
this.clientConfiguration.option(SdkClientOption.ENDPOINT_PROVIDER, endpointProvider);
28+
return this;
29+
}
30+
1931
}

0 commit comments

Comments
 (0)