Fips compliance internal

CHANGELOG-3.0.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
+- Support for FIPS-140-3 compliance through environment variable ([#3420](https://github.com/opensearch-project/OpenSearch/pull/14912))
 
 ### Dependencies
 

build.gradle

@@ -65,7 +65,6 @@ apply from: 'gradle/ide.gradle'
 apply from: 'gradle/forbidden-dependencies.gradle'
 apply from: 'gradle/formatting.gradle'
 apply from: 'gradle/local-distribution.gradle'
-apply from: 'gradle/fips.gradle'
 apply from: 'gradle/run.gradle'
 apply from: 'gradle/missing-javadoc.gradle'
 apply from: 'gradle/code-coverage.gradle'
@@ -472,8 +471,8 @@ gradle.projectsEvaluated {
   }
 }
 
-// test retry configuration
 subprojects {
+  // test retry configuration
   tasks.withType(Test).configureEach {
     develocity.testRetry {
       if (BuildParams.isCi()) {
@@ -559,6 +558,15 @@ subprojects {
       }
     }
   }
+
+  // test with FIPS-140-3 enabled
+  plugins.withType(JavaPlugin).configureEach {
+    tasks.withType(Test).configureEach { testTask ->
+      if (System.getenv('OPENSEARCH_CRYPTO_STANDARD') == 'FIPS-140-3') {
+        testTask.jvmArgs += "-Dorg.bouncycastle.fips.approved_only=true"
+      }
+    }
+  }
 }
 
 // eclipse configuration

buildSrc/build.gradle

@@ -123,6 +123,8 @@ dependencies {
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.2.1"
+  api "org.bouncycastle:bc-fips:${props.getProperty('bouncycastle_jce')}"
+
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
@@ -229,12 +231,8 @@ if (project != rootProject) {
 
   forbiddenPatterns {
     exclude '**/*.wav'
-    exclude '**/*.p12'
-    exclude '**/*.jks'
-    exclude '**/*.crt'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
-    exclude '**/*.bcfks'
   }
 
   testingConventions {

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -164,6 +164,12 @@ public void execute(Task t) {
                 test.systemProperty("tests.seed", BuildParams.getTestSeed());
             }
 
+            var securityFile = BuildParams.isInFipsJvm() ? "fips_java.security" : "java.security";
+            test.systemProperty(
+                "java.security.properties",
+                project.getRootProject().getLayout().getProjectDirectory() + "/distribution/src/config/" + securityFile
+            );
+
             // don't track these as inputs since they contain absolute paths and break cache relocatability
             File gradleHome = project.getGradle().getGradleUserHomeDir();
             String gradleVersion = project.getGradle().getGradleVersion();

buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java

@@ -32,8 +32,10 @@
 
 package org.opensearch.gradle.http;
 
+import org.bouncycastle.crypto.CryptoServicesRegistrar;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.internal.impldep.com.jcraft.jsch.annotations.SuppressForbiddenApi;
 
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.KeyManager;
@@ -51,7 +53,6 @@
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
-import java.security.SecureRandom;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateFactory;
 import java.util.Arrays;
@@ -216,15 +217,15 @@ KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
     }
 
     private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
-        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        var keyStore = getKeyStoreInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
         try (InputStream input = new FileInputStream(trustStoreFile)) {
             keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
         }
         return keyStore;
     }
 
     private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
-        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        var store = getKeyStoreInstance(KeyStore.getDefaultType());
         store.load(null, null);
         final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
         int counter = 0;
@@ -239,12 +240,17 @@ private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOExce
         return store;
     }
 
+    @SuppressForbiddenApi("runs exclusively in test-context without KeyStoreFactory on classpath.")
+    private KeyStore getKeyStoreInstance(String type) throws KeyStoreException {
+        return KeyStore.getInstance(type);
+    }
+
     private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
         checkForTrustEntry(trustStore);
         TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
         tmf.init(trustStore);
         SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
+        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), CryptoServicesRegistrar.getSecureRandom());
         return sslContext;
     }
 

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -77,6 +77,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
     private static final String DEFAULT_LEGACY_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/LegacyESVersion.java";
     private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/Version.java";
+    protected static final String OPENSEARCH_CRYPTO_STANDARD = "OPENSEARCH_CRYPTO_STANDARD";
     private static Integer _defaultParallel = null;
 
     private final JvmMetadataDetector jvmMetadataDetector;
@@ -112,6 +113,8 @@ public void apply(Project project) {
         BuildParams.init(params -> {
             // Initialize global build parameters
             boolean isInternal = GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null;
+            var cryptoStandard = System.getenv(OPENSEARCH_CRYPTO_STANDARD);
+            var inFipsJvm = cryptoStandard != null && cryptoStandard.equals("FIPS-140-3");
 
             params.reset();
             params.setRuntimeJavaHome(runtimeJavaHome);
@@ -129,7 +132,7 @@ public void apply(Project project) {
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(isInternal);
             params.setDefaultParallel(findDefaultParallel(project));
-            params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
+            params.setInFipsJvm(inFipsJvm);
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
             if (isInternal) {
                 params.setBwcVersions(resolveBwcVersions(rootDir));
@@ -163,6 +166,7 @@ private void logGlobalBuildInfo() {
         final String osArch = System.getProperty("os.arch");
         final Jvm gradleJvm = Jvm.current();
         final String gradleJvmDetails = getJavaInstallation(gradleJvm.getJavaHome()).getDisplayName();
+        final String cryptStandard = System.getenv(OPENSEARCH_CRYPTO_STANDARD);
 
         LOGGER.quiet("=======================================");
         LOGGER.quiet("OpenSearch Build Hamster says Hello!");
@@ -179,7 +183,11 @@ private void logGlobalBuildInfo() {
             LOGGER.quiet("  JAVA_HOME             : " + gradleJvm.getJavaHome());
         }
         LOGGER.quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
-        LOGGER.quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
+        if (cryptStandard != null && cryptStandard.equals("FIPS-140-3")) {
+            LOGGER.quiet("  Crypto Standard       : FIPS-140-3");
+        } else {
+            LOGGER.quiet("  Crypto Standard       : any-supported");
+        }
         LOGGER.quiet("=======================================");
     }
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -80,8 +80,13 @@ public class ForbiddenPatternsTask extends DefaultTask {
         .exclude("**/*.ico")
         .exclude("**/*.jar")
         .exclude("**/*.zip")
+        .exclude("**/*.p12")
         .exclude("**/*.jks")
         .exclude("**/*.crt")
+        .exclude("**/*.der")
+        .exclude("**/*.pem")
+        .exclude("**/*.key")
+        .exclude("**/*.bcfks")
         .exclude("**/*.keystore")
         .exclude("**/*.png");
 

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -548,15 +548,15 @@ public synchronized void start() {
 
         logToProcessStdout("Creating opensearch keystore with password set to [" + keystorePassword + "]");
         if (keystorePassword.length() > 0) {
-            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword, "opensearch-keystore", "create", "-p");
+            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword + "\n", "opensearch-keystore", "create", "-p");
         } else {
             runOpenSearchBinScript("opensearch-keystore", "-v", "create");
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
 
-            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", "-x", key));
+            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", key));
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
                 File file = entry.getValue();
@@ -738,7 +738,12 @@ private void runOpenSearchBinScriptWithInput(String input, String tool, CharSequ
     }
 
     private void runKeystoreCommandWithPassword(String keystorePassword, String input, CharSequence... args) {
-        final String actualInput = keystorePassword.length() > 0 ? keystorePassword + "\n" + input : input;
+        final String actualInput;
+        if (keystorePassword.length() > 0) {
+            actualInput = keystorePassword + "\n" + input + "\n" + input;
+        } else {
+            actualInput = input + "\n" + input;
+        }
         runOpenSearchBinScriptWithInput(actualInput, "opensearch-keystore", args);
     }