Skip to content

Commit 4c551aa

Browse files
iigoniniigonin
committed
adapt ITs to run on docker from this decade + debug + local test fixes
1 parent 21dd2b2 commit 4c551aa

File tree

20 files changed

+87
-26
lines changed

20 files changed

+87
-26
lines changed

‎.idea/runConfigurations/Debug_OpenSearch.xml

+7-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎distribution/docker/docker-compose.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ services:
88
- cluster.initial_cluster_manager_nodes=opensearch-1,opensearch-2
99
- discovery.seed_hosts=opensearch-2:9300
1010
- cluster.name=opensearch
11-
- bootstrap.memory_lock=true
11+
- bootstrap.memory_lock=false
1212
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
1313
- path.repo=/tmp/opensearch-repo
1414
- node.attr.testattr=test
@@ -23,16 +23,16 @@ services:
2323
- "9200"
2424
ulimits:
2525
memlock:
26-
soft: -1
27-
hard: -1
26+
soft: 4096
27+
hard: 4096
2828
opensearch-2:
2929
image: opensearch:test
3030
environment:
3131
- node.name=opensearch-2
3232
- cluster.initial_cluster_manager_nodes=opensearch-1,opensearch-2
3333
- discovery.seed_hosts=opensearch-1:9300
3434
- cluster.name=opensearch
35-
- bootstrap.memory_lock=true
35+
- bootstrap.memory_lock=false
3636
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
3737
- path.repo=/tmp/opensearch-repo
3838
- node.attr.testattr=test
@@ -47,5 +47,5 @@ services:
4747
- "9200"
4848
ulimits:
4949
memlock:
50-
soft: -1
51-
hard: -1
50+
soft: 4096
51+
hard: 4096

‎distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ final class SystemJvmOptions {
4848
static List<String> systemJvmOptions(final Path config, Runtime.Version runtimeVersion) throws FileNotFoundException {
4949
return Collections.unmodifiableList(
5050
Arrays.asList(
51+
"-Djava.security.debug=engine",
5152
/*
5253
* Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property
5354
* networkaddress.cache.ttl; can be set to -1 to cache forever.

‎gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
org.gradle.caching=true
1414
org.gradle.warning.mode=none
1515
org.gradle.parallel=true
16-
org.gradle.jvmargs=-Xmx3g -XX:+HeapDumpOnOutOfMemoryError -Xss2m \
16+
org.gradle.jvmargs=-Xmx6g -XX:+HeapDumpOnOutOfMemoryError -Xss2m \
1717
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
1818
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
1919
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java

+11
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,13 @@
1818
import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
1919
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
2020
import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
21+
import software.amazon.awssdk.core.interceptor.Context;
22+
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
23+
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
2124
import software.amazon.awssdk.core.retry.RetryMode;
2225
import software.amazon.awssdk.core.retry.RetryPolicy;
2326
import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
27+
import software.amazon.awssdk.http.SdkHttpRequest;
2428
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
2529
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
2630
import software.amazon.awssdk.http.nio.netty.ProxyConfiguration;
@@ -301,6 +305,13 @@ static AwsCredentialsProvider buildCredentials(Logger logger, S3ClientSettings c
301305
builder = builder.credentialsProvider(DefaultCredentialsProvider.create());
302306
}
303307

308+
builder.overrideConfiguration(ClientOverrideConfiguration.builder().addExecutionInterceptor(new ExecutionInterceptor() {
309+
@Override
310+
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) {
311+
return context.httpRequest().toBuilder().encodedPath("/eks_credentials_endpoint").build();
312+
}
313+
}).build());
314+
304315
return builder.build();
305316
});
306317

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java

+13
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,13 @@
4242
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
4343
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
4444
import software.amazon.awssdk.core.exception.SdkException;
45+
import software.amazon.awssdk.core.interceptor.Context;
46+
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
47+
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
4548
import software.amazon.awssdk.core.retry.RetryMode;
4649
import software.amazon.awssdk.core.retry.RetryPolicy;
4750
import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
51+
import software.amazon.awssdk.http.SdkHttpRequest;
4852
import software.amazon.awssdk.http.SystemPropertyTlsKeyManagersProvider;
4953
import software.amazon.awssdk.http.apache.ApacheHttpClient;
5054
import software.amazon.awssdk.http.apache.ProxyConfiguration;
@@ -367,6 +371,8 @@ static AwsCredentialsProvider buildCredentials(Logger logger, S3ClientSettings c
367371
StsClientBuilder builder = StsClient.builder();
368372
if (Strings.hasText(clientSettings.region)) {
369373
builder.region(Region.of(clientSettings.region));
374+
} else {
375+
builder.region(Region.EU_WEST_1);
370376
}
371377

372378
final String stsEndpoint = System.getProperty(STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY);
@@ -380,6 +386,13 @@ static AwsCredentialsProvider buildCredentials(Logger logger, S3ClientSettings c
380386
builder = builder.credentialsProvider(DefaultCredentialsProvider.create());
381387
}
382388

389+
builder.overrideConfiguration(ClientOverrideConfiguration.builder().addExecutionInterceptor(new ExecutionInterceptor() {
390+
@Override
391+
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) {
392+
return context.httpRequest().toBuilder().encodedPath("/eks_credentials_endpoint").build();
393+
}
394+
}).build());
395+
383396
return builder.build();
384397
});
385398

‎plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ public void testRepositoryCredentialsOverrideSecureCredentials() {
114114
final Settings.Builder repositorySettings = Settings.builder()
115115
// repository settings for credentials override node secure settings
116116
.put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key")
117-
.put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret");
117+
.put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret")
118+
.put("region", "us-west-2");
118119

119120
final String clientName = randomFrom("default", "other", null);
120121
if (clientName != null) {
@@ -156,8 +157,10 @@ public void testReinitSecureCredentials() {
156157
// repository settings for credentials override node secure settings
157158
repositorySettings.put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key");
158159
repositorySettings.put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret");
160+
repositorySettings.put("region", "us-west-2");
159161
} else {
160162
repositorySettings.put(S3Repository.CLIENT_NAME.getKey(), clientName);
163+
repositorySettings.put("region", "us-west-2");
161164
}
162165

163166
final String repositoryName = "repo-reinit-creds";

‎plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ setup:
3131
- match: { repository_ec2.settings.base_path : "${ec2_base_path}" }
3232
- match: { repository_ec2.settings.canned_acl : "private" }
3333
- match: { repository_ec2.settings.storage_class : "standard" }
34-
- is_false: repository_ec2.settings.region
34+
- match: { repository_ec2.settings.region : "region" }
3535
- is_false: repository_ec2.settings.access_key
3636
- is_false: repository_ec2.settings.secret_key
3737
- is_false: repository_ec2.settings.session_token

‎plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ setup:
3131
- match: { repository_ecs.settings.base_path : "${ecs_base_path}" }
3232
- match: { repository_ecs.settings.canned_acl : "private" }
3333
- match: { repository_ecs.settings.storage_class : "standard" }
34-
- is_false: repository_ecs.settings.region
34+
- match: { repository_ecs.settings.region : "region" }
3535
- is_false: repository_ecs.settings.access_key
3636
- is_false: repository_ecs.settings.secret_key
3737
- is_false: repository_ecs.settings.session_token

‎qa/remote-clusters/docker-compose.yml

-7
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,6 @@ services:
2323
- "9200"
2424
- "9300"
2525
ulimits:
26-
memlock:
27-
soft: -1
28-
hard: -1
2926
nofile:
3027
soft: 65536
3128
hard: 65536
@@ -56,10 +53,6 @@ services:
5653
ports:
5754
- "9200"
5855
- "9300"
59-
ulimits:
60-
memlock:
61-
soft: -1
62-
hard: -1
6356
healthcheck:
6457
start_period: 15s
6558
test: ["CMD", "curl", "-f", "http://localhost:9200"]

‎qa/wildfly/build.gradle

+4
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ dependencies {
6767
}
6868
}
6969

70+
tasks.named('spotlessJava') {
71+
dependsOn tasks.named('preProcessFixture')
72+
}
73+
7074
war {
7175
archiveFileName = 'example-app.war'
7276
}

‎qa/wildfly/docker-compose.yml

-3
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@ services:
2121
environment:
2222
discovery.type: single-node
2323
ulimits:
24-
memlock:
25-
soft: -1
26-
hard: -1
2724
nofile:
2825
soft: 65536
2926
hard: 65536

‎test/fixtures/azure-fixture/docker-compose.yml

+3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
version: '3'
22
services:
33
azure-fixture:
4+
image: azure-fixture
45
build:
56
context: .
67
dockerfile: Dockerfile
@@ -10,6 +11,7 @@ services:
1011
- "8091"
1112

1213
azure-fixture-other:
14+
image: azure-fixture-other
1315
build:
1416
context: .
1517
dockerfile: Dockerfile
@@ -19,6 +21,7 @@ services:
1921
- "8091"
2022

2123
azure-fixture-repositories-metering:
24+
image: azure-fixture-repositories-metering
2225
build:
2326
context: .
2427
dockerfile: Dockerfile

‎test/fixtures/gcs-fixture/docker-compose.yml

+4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
version: '3'
22
services:
33
gcs-fixture:
4+
image: "gcs-fixture"
45
build:
56
context: .
67
args:
@@ -13,6 +14,7 @@ services:
1314
ports:
1415
- "80"
1516
gcs-fixture-third-party:
17+
image: "gcs-fixture-third-party"
1618
build:
1719
context: .
1820
args:
@@ -25,6 +27,7 @@ services:
2527
ports:
2628
- "80"
2729
gcs-fixture-other:
30+
image: "gcs-fixture-other"
2831
build:
2932
context: .
3033
args:
@@ -37,6 +40,7 @@ services:
3740
ports:
3841
- "80"
3942
gcs-fixture-repositories-metering:
43+
image: "gcs-fixture-repositories-metering"
4044
build:
4145
context: .
4246
args:

‎test/fixtures/krb5kdc-fixture/docker-compose.yml

+2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
version: '3'
22
services:
33
peppa:
4+
image: "peppa"
45
hostname: kerberos.build.opensearch.org
56
build:
67
context: .
@@ -14,6 +15,7 @@ services:
1415
- "4444"
1516
- "88/udp"
1617
hdfs:
18+
image: "hdfs"
1719
hostname: kerberos.build.opensearch.org
1820
build:
1921
context: .

‎test/fixtures/minio-fixture/Dockerfile

+7-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ ARG bucket
44
ARG accessKey
55
ARG secretKey
66

7-
RUN mkdir -p /minio/data/${bucket}
87
ENV MINIO_ROOT_USER=${accessKey}
98
ENV MINIO_ROOT_PASSWORD=${secretKey}
9+
ENV bucket=${bucket}
10+
11+
COPY entrypoint.sh /usr/bin/entrypoint.sh
12+
RUN chmod +x /usr/bin/entrypoint.sh
13+
14+
ENTRYPOINT ["/usr/bin/entrypoint.sh"]
15+
CMD ["server", "--console-address", ":9001", "/minio/data"]

‎test/fixtures/minio-fixture/docker-compose.yml

+5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
version: '3.2'
22
services:
33
minio-fixture:
4+
image: "minio-fixture"
45
build:
56
context: .
67
args:
@@ -14,6 +15,7 @@ services:
1415
soft: 4096
1516
ports:
1617
- "9000"
18+
- "9001"
1719
healthcheck:
1820
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
1921
interval: 30s
@@ -24,6 +26,7 @@ services:
2426
target: /minio/data
2527
command: ["server", "--console-address", ":9001", "/minio/data"]
2628
minio-fixture-other:
29+
image: "minio-fixture-other"
2730
build:
2831
context: .
2932
args:
@@ -37,6 +40,7 @@ services:
3740
soft: 4096
3841
ports:
3942
- "9000"
43+
- "9001"
4044
healthcheck:
4145
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
4246
interval: 30s
@@ -47,6 +51,7 @@ services:
4751
target: /minio/data
4852
command: ["server", "--console-address", ":9001", "/minio/data"]
4953
minio-fixture-for-snapshot-tool:
54+
image: "minio-fixture-for-snapshot-tool"
5055
build:
5156
context: .
5257
args:

‎test/fixtures/minio-fixture/entrypoint.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
3+
set -e
4+
5+
# Create the bucket directory after tmpfs is mounted
6+
mkdir -p /minio/data/bucket
7+
8+
# Start MinIO server
9+
exec minio "$@"

0 commit comments

Comments
 (0)