Skip to content

Commit 90f7337

Browse files
guojialiang92guojialiang92
committed
Merge remote-tracking branch 'origin/main' into dev/add_async_publish_checkpoint_task
2 parents 89574de + d066743 commit 90f7337

File tree

248 files changed

+15047
-563
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

248 files changed

+15047
-563
lines changed

‎.github/workflows/benchmark-pull-request.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ jobs:
167167
- name: Trigger jenkins workflow to run gradle check
168168
run: |
169169
cat $GITHUB_ENV
170-
bash opensearch-build/scripts/benchmark/benchmark-pull-request.sh ${{ secrets.JENKINS_PR_BENCHMARK_GENERIC_WEBHOOK_TOKEN }}
170+
bash opensearch-build/scripts/benchmark/benchmark-pull-request.sh -t ${{ secrets.JENKINS_PR_BENCHMARK_GENERIC_WEBHOOK_TOKEN }} -u ${{ secrets.JENKINS_GITHUB_USER}} -p ${{ secrets.JENKINS_GITHUB_USER_TOKEN}}
171171
- name: Update PR with Job Url
172172
uses: actions/github-script@v7
173173
with:

‎.github/workflows/gradle-check.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
- uses: actions/checkout@v4
2525
- name: Get changed files
2626
id: changed-files-specific
27-
uses: tj-actions/[email protected].1
27+
uses: tj-actions/[email protected].3
2828
with:
2929
files_ignore: |
3030
release-notes/*.md
@@ -103,7 +103,7 @@ jobs:
103103
run: |
104104
set -e
105105
set -o pipefail
106-
bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
106+
bash opensearch-build/scripts/gradle/gradle-check.sh -t ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} -u ${{ secrets.JENKINS_GITHUB_USER}} -p ${{ secrets.JENKINS_GITHUB_USER_TOKEN}} | tee -a gradle-check.log
107107
108108
- name: Setup Result Status
109109
if: always()

‎CHANGELOG.md

+9-1
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1111
- Introduce a new search node role to hold search only shards ([#17620](https://github.com/opensearch-project/OpenSearch/pull/17620))
1212
- Fix systemd integTest on deb regarding path ownership check ([#17641](https://github.com/opensearch-project/OpenSearch/pull/17641))
1313
- Add dfs transformation function in XContentMapValues ([#17612](https://github.com/opensearch-project/OpenSearch/pull/17612))
14+
- [Security Manager Replacement] Add support of Java policies ([#17663](https://github.com/opensearch-project/OpenSearch/pull/17663))
15+
- Added Kinesis support as a plugin for the pull-based ingestion ([#17615](https://github.com/opensearch-project/OpenSearch/pull/17615)
16+
17+
### Changed
18+
- Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912))
1419

1520
### Dependencies
16-
- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 10.0.2 ([#17607](https://github.com/opensearch-project/OpenSearch/pull/17607))
21+
- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 10.0.2 ([#17607](https://github.com/opensearch-project/OpenSearch/pull/17607), [#17669](https://github.com/opensearch-project/OpenSearch/pull/17669))
1722
- Bump `com.google.api:api-common` from 1.8.1 to 2.46.1 ([#17604](https://github.com/opensearch-project/OpenSearch/pull/17604))
1823
- Bump `ch.qos.logback:logback-core` from 1.5.16 to 1.5.17 ([#17609](https://github.com/opensearch-project/OpenSearch/pull/17609))
1924
- Bump `org.jruby.joni:joni` from 2.2.3 to 2.2.5 ([#17608](https://github.com/opensearch-project/OpenSearch/pull/17608))
2025
- Bump `dangoslen/dependabot-changelog-helper` from 3 to 4 ([#17498](https://github.com/opensearch-project/OpenSearch/pull/17498))
2126
- Bump `com.google.api:gax` from 2.35.0 to 2.63.1 ([#17465](https://github.com/opensearch-project/OpenSearch/pull/17465))
27+
- Bump `com.azure:azure-storage-blob` from 12.29.1 to 12.30.0 ([#17667](https://github.com/opensearch-project/OpenSearch/pull/17667))
28+
- Bump `tj-actions/changed-files` from 46.0.1 to 46.0.3 ([#17666](https://github.com/opensearch-project/OpenSearch/pull/17666))
29+
- Bump `com.google.code.gson:gson` from 2.11.0 to 2.12.1 ([#17668](https://github.com/opensearch-project/OpenSearch/pull/17668))
2230

2331
### Changed
2432

‎buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

+6
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ public void execute(Task t) {
164164
test.systemProperty("tests.seed", BuildParams.getTestSeed());
165165
}
166166

167+
var securityFile = "java.security";
168+
test.systemProperty(
169+
"java.security.properties",
170+
project.getRootProject().getLayout().getProjectDirectory() + "/distribution/src/config/" + securityFile
171+
);
172+
167173
// don't track these as inputs since they contain absolute paths and break cache relocatability
168174
File gradleHome = project.getGradle().getGradleUserHomeDir();
169175
String gradleVersion = project.getGradle().getGradleVersion();

‎client/rest/build.gradle

+7
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ dependencies {
5151
api "commons-codec:commons-codec:${versions.commonscodec}"
5252
api "commons-logging:commons-logging:${versions.commonslogging}"
5353
api "org.slf4j:slf4j-api:${versions.slf4j}"
54+
runtimeOnly "org.bouncycastle:bc-fips:${versions.bouncycastle_jce}"
55+
runtimeOnly "org.bouncycastle:bctls-fips:${versions.bouncycastle_tls}"
56+
runtimeOnly "org.bouncycastle:bcutil-fips:${versions.bouncycastle_util}"
5457

5558
// reactor
5659
api "io.projectreactor:reactor-core:${versions.reactor}"
@@ -70,6 +73,10 @@ dependencies {
7073
testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
7174
}
7275

76+
tasks.named("dependencyLicenses").configure {
77+
mapping from: /bc.*/, to: 'bouncycastle'
78+
}
79+
7380
tasks.withType(CheckForbiddenApis).configureEach {
7481
//client does not depend on server, so only jdk and http signatures should be checked
7582
replaceSignatureFiles('jdk-signatures', 'http-signatures')

‎client/rest/licenses/bc-fips-2.0.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ee9ac432cf08f9a9ebee35d7cf8a45f94959a7ab

‎client/rest/licenses/bctls-fips-2.0.19.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
9cc33650ede63bc1a8281ed5c8e1da314d50bc76

‎client/rest/licenses/bcutil-fips-2.0.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a1857cd639295b10cc90e6d31ecbc523cdafcc19

‎client/rest/licenses/bouncycastle-LICENSE.txt

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
Copyright (c) 2000 - 2023 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
4+
documentation files (the "Software"), to deal in the Software without restriction, including without limitation
5+
the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
6+
and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
7+
8+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
9+
Software.
10+
11+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
12+
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
13+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
14+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎client/rest/licenses/bouncycastle-NOTICE.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java

+33-34
Original file line numberDiff line numberDiff line change
@@ -38,27 +38,23 @@
3838
import com.sun.net.httpserver.HttpsServer;
3939

4040
import org.apache.hc.core5.http.HttpHost;
41+
import org.apache.hc.core5.ssl.SSLContextBuilder;
4142
import org.junit.AfterClass;
4243
import org.junit.BeforeClass;
4344

4445
import javax.net.ssl.KeyManagerFactory;
4546
import javax.net.ssl.SSLContext;
46-
import javax.net.ssl.SSLHandshakeException;
47+
import javax.net.ssl.SSLException;
4748
import javax.net.ssl.TrustManagerFactory;
4849

4950
import java.io.IOException;
5051
import java.io.InputStream;
5152
import java.net.InetAddress;
5253
import java.net.InetSocketAddress;
53-
import java.nio.file.Files;
54-
import java.nio.file.Paths;
5554
import java.security.AccessController;
56-
import java.security.KeyFactory;
5755
import java.security.KeyStore;
5856
import java.security.PrivilegedAction;
59-
import java.security.cert.Certificate;
60-
import java.security.cert.CertificateFactory;
61-
import java.security.spec.PKCS8EncodedKeySpec;
57+
import java.security.SecureRandom;
6258

6359
import static org.hamcrest.Matchers.instanceOf;
6460
import static org.junit.Assert.assertEquals;
@@ -75,7 +71,7 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
7571
@BeforeClass
7672
public static void startHttpServer() throws Exception {
7773
httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
78-
httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext()));
74+
httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext(true)));
7975
httpsServer.createContext("/", new ResponseHandler());
8076
httpsServer.start();
8177
}
@@ -103,11 +99,11 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
10399
client.performRequest(new Request("GET", "/"));
104100
fail("connection should have been rejected due to SSL handshake");
105101
} catch (Exception e) {
106-
assertThat(e, instanceOf(SSLHandshakeException.class));
102+
assertThat(e.getCause(), instanceOf(SSLException.class));
107103
}
108104
}
109105

110-
SSLContext.setDefault(getSslContext());
106+
SSLContext.setDefault(getSslContext(false));
111107
try (RestClient client = buildRestClient()) {
112108
Response response = client.performRequest(new Request("GET", "/"));
113109
assertEquals(200, response.getStatusLine().getStatusCode());
@@ -122,34 +118,37 @@ private RestClient buildRestClient() {
122118
return RestClient.builder(new HttpHost("https", address.getHostString(), address.getPort())).build();
123119
}
124120

125-
private static SSLContext getSslContext() throws Exception {
126-
SSLContext sslContext = SSLContext.getInstance(getProtocol());
121+
private static SSLContext getSslContext(boolean server) throws Exception {
122+
SSLContext sslContext;
123+
char[] password = "password".toCharArray();
124+
SecureRandom secureRandom = SecureRandom.getInstance("DEFAULT", "BCFIPS");
125+
String fileExtension = ".jks";
126+
127127
try (
128-
InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt");
129-
InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore.jks")
128+
InputStream trustStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test_truststore" + fileExtension);
129+
InputStream keyStoreFile = RestClientBuilderIntegTests.class.getResourceAsStream("/testks" + fileExtension)
130130
) {
131-
// Build a keystore of default type programmatically since we can't use JKS keystores to
132-
// init a KeyManagerFactory in FIPS 140 JVMs.
133-
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
134-
keyStore.load(null, "password".toCharArray());
135-
CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
136-
PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(
137-
Files.readAllBytes(Paths.get(RestClientBuilderIntegTests.class.getResource("/test.der").toURI()))
138-
);
139-
KeyFactory keyFactory = KeyFactory.getInstance("RSA");
140-
keyStore.setKeyEntry(
141-
"mykey",
142-
keyFactory.generatePrivate(privateKeySpec),
143-
"password".toCharArray(),
144-
new Certificate[] { certFactory.generateCertificate(certFile) }
145-
);
146-
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
147-
kmf.init(keyStore, "password".toCharArray());
131+
KeyStore keyStore = KeyStore.getInstance("JKS");
132+
keyStore.load(keyStoreFile, password);
133+
KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
134+
kmf.init(keyStore, password);
135+
148136
KeyStore trustStore = KeyStore.getInstance("JKS");
149-
trustStore.load(keyStoreFile, "password".toCharArray());
150-
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
137+
trustStore.load(trustStoreFile, password);
138+
TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX", "BCJSSE");
151139
tmf.init(trustStore);
152-
sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
140+
141+
SSLContextBuilder sslContextBuilder = SSLContextBuilder.create()
142+
.setProvider("BCJSSE")
143+
.setProtocol(getProtocol())
144+
.setSecureRandom(secureRandom);
145+
146+
if (server) {
147+
sslContextBuilder.loadKeyMaterial(keyStore, password);
148+
}
149+
sslContextBuilder.loadTrustMaterial(trustStore, null);
150+
sslContext = sslContextBuilder.build();
151+
153152
}
154153
return sslContext;
155154
}

‎distribution/src/config/java.security

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Security properties for non-approved mode 'org.bouncycastle.fips.approved_only=false'.
2+
# Intended to be used complementary with a single equal sign e.g. 'java.security.properties=java.security'
3+
4+
security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider
5+
security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider
6+
security.provider.3=SUN
7+
security.provider.4=SunJGSS
8+
9+
ssl.KeyManagerFactory.algorithm=PKIX
10+
ssl.TrustManagerFactory.algorithm=PKIX

‎distribution/tools/keystore-cli/src/test/java/org/opensearch/tools/cli/keystore/AddFileKeyStoreCommandTests.java

+7-11
Original file line numberDiff line numberDiff line change
@@ -211,17 +211,13 @@ public void testIncorrectPassword() throws Exception {
211211
terminal.addSecretInput("thewrongkeystorepassword");
212212
UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString()));
213213
assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
214-
if (inFipsJvm()) {
215-
assertThat(
216-
e.getMessage(),
217-
anyOf(
218-
containsString("Provided keystore password was incorrect"),
219-
containsString("Keystore has been corrupted or tampered with")
220-
)
221-
);
222-
} else {
223-
assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
224-
}
214+
assertThat(
215+
e.getMessage(),
216+
anyOf(
217+
containsString("Provided keystore password was incorrect"),
218+
containsString("Keystore has been corrupted or tampered with")
219+
)
220+
);
225221
}
226222

227223
public void testAddToUnprotectedKeystore() throws Exception {

‎distribution/tools/keystore-cli/src/test/java/org/opensearch/tools/cli/keystore/AddStringKeyStoreCommandTests.java

+7-12
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,13 @@ public void testInvalidPassphrease() throws Exception {
7272
terminal.addSecretInput("thewrongpassword");
7373
UserException e = expectThrows(UserException.class, () -> execute("foo2"));
7474
assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
75-
if (inFipsJvm()) {
76-
assertThat(
77-
e.getMessage(),
78-
anyOf(
79-
containsString("Provided keystore password was incorrect"),
80-
containsString("Keystore has been corrupted or tampered with")
81-
)
82-
);
83-
} else {
84-
assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
85-
}
86-
75+
assertThat(
76+
e.getMessage(),
77+
anyOf(
78+
containsString("Provided keystore password was incorrect"),
79+
containsString("Keystore has been corrupted or tampered with")
80+
)
81+
);
8782
}
8883

8984
public void testMissingPromptCreateWithoutPasswordWhenPrompted() throws Exception {

‎distribution/tools/keystore-cli/src/test/java/org/opensearch/tools/cli/keystore/ChangeKeyStorePasswordCommandTests.java

+7-11
Original file line numberDiff line numberDiff line change
@@ -104,16 +104,12 @@ public void testChangeKeyStorePasswordWrongExistingPassword() throws Exception {
104104
// We'll only be prompted once (for the old password)
105105
UserException e = expectThrows(UserException.class, this::execute);
106106
assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
107-
if (inFipsJvm()) {
108-
assertThat(
109-
e.getMessage(),
110-
anyOf(
111-
containsString("Provided keystore password was incorrect"),
112-
containsString("Keystore has been corrupted or tampered with")
113-
)
114-
);
115-
} else {
116-
assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
117-
}
107+
assertThat(
108+
e.getMessage(),
109+
anyOf(
110+
containsString("Provided keystore password was incorrect"),
111+
containsString("Keystore has been corrupted or tampered with")
112+
)
113+
);
118114
}
119115
}

0 commit comments

Comments
 (0)