Merge branch 'main' into fips_compliance2

Author: iigonin

.github/workflows/dependabot_pr.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Update the changelog
         uses: dangoslen/dependabot-changelog-helper@v4
         with:
-          version: 'Unreleased 2.x'
+          version: 'Unreleased 3.x'
 
       - name: Commit the changes
         uses: stefanzweifel/git-auto-commit-action@v5

CHANGELOG.md

@@ -6,16 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 3.x]
 ### Added
 - Change priority for scheduling reroute during timeout([#16445](https://github.com/opensearch-project/OpenSearch/pull/16445))
+- [Rule Based Auto-tagging] Add rule schema for auto tagging ([#17238](https://github.com/opensearch-project/OpenSearch/pull/17238))
 - Renaming the node role search to warm ([#17573](https://github.com/opensearch-project/OpenSearch/pull/17573))
 - Introduce a new search node role to hold search only shards ([#17620](https://github.com/opensearch-project/OpenSearch/pull/17620))
+- Fix systemd integTest on deb regarding path ownership check ([#17641](https://github.com/opensearch-project/OpenSearch/pull/17641)) 
+- Add dfs transformation function in XContentMapValues ([#17612](https://github.com/opensearch-project/OpenSearch/pull/17612))
+
+### Changed
+- Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912))
 
 ### Changed
 - Migrate BC libs to their FIPS counterparts ([#3420](https://github.com/opensearch-project/OpenSearch/pull/14912))
 
 ### Dependencies
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 10.0.2 ([#17607](https://github.com/opensearch-project/OpenSearch/pull/17607))
+- Bump `com.google.api:api-common` from 1.8.1 to 2.46.1 ([#17604](https://github.com/opensearch-project/OpenSearch/pull/17604))
 - Bump `ch.qos.logback:logback-core` from 1.5.16 to 1.5.17 ([#17609](https://github.com/opensearch-project/OpenSearch/pull/17609))
 - Bump `org.jruby.joni:joni` from 2.2.3 to 2.2.5 ([#17608](https://github.com/opensearch-project/OpenSearch/pull/17608))
 - Bump `dangoslen/dependabot-changelog-helper` from 3 to 4 ([#17498](https://github.com/opensearch-project/OpenSearch/pull/17498))
+- Bump `com.google.api:gax` from 2.35.0 to 2.63.1 ([#17465](https://github.com/opensearch-project/OpenSearch/pull/17465))
 
 ### Changed
 

benchmarks/src/main/java20/org/opensearch/common/round/RoundableSupplier.java

@@ -29,6 +29,7 @@
  */
 
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
+import org.opensearch.gradle.precommit.TestingConventionsTasks
 
 apply plugin: 'opensearch.build'
 apply plugin: 'opensearch.publish'

client/rest/build.gradle

@@ -103,7 +103,7 @@ public void makeRequest(KeyStoreType keyStoreType) throws Exception {
             try (RestClient client = buildRestClient()) {
                 try {
                     client.performRequest(new Request("GET", "/"));
-                    fail("connection should have been rejected due to SSL failure");
+                    fail("connection should have been rejected due to SSL handshake");
                 } catch (Exception e) {
                     assertThat(e.getCause(), instanceOf(SSLException.class));
                 }

client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java

@@ -528,77 +528,6 @@ private byte[] generateV2() throws Exception {
         return fileBytes;
     }
 
-    private void generateV1() throws IOException, NoSuchAlgorithmException, NoSuchProviderException, CertificateException,
-        InvalidKeySpecException, KeyStoreException {
-        Path configDir = env.configDir();
-        NIOFSDirectory directory = new NIOFSDirectory(configDir);
-        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
-            CodecUtil.writeHeader(output, "opensearch.keystore", 1);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
-            KeyStore keystore = KeyStoreFactory.getInstance(KeyStoreType.PKCS_12, "SUN");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-    }
-
-    private byte[] generateV2() throws Exception {
-        Path configDir = env.configDir();
-        NIOFSDirectory directory = new NIOFSDirectory(configDir);
-        byte[] fileBytes = new byte[20];
-        random().nextBytes(fileBytes);
-        try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, "opensearch.keystore", IOContext.DEFAULT)) {
-
-            CodecUtil.writeHeader(output, "opensearch.keystore", 2);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE"); // string algo
-            output.writeString("PBE"); // file algo
-
-            output.writeVInt(2); // num settings
-            output.writeString("string_setting");
-            output.writeString("STRING");
-            output.writeString("file_setting");
-            output.writeString("FILE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE", "SunJCE");
-            KeyStore keystore = KeyStoreFactory.getInstance(KeyStoreType.PKCS_12, "SUN");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
-            char[] chars = new char[base64Bytes.length];
-            for (int i = 0; i < chars.length; ++i) {
-                chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
-            }
-            secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
-            keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-
-        return fileBytes;
-    }
-
     private byte[] toByteArray(final InputStream is) throws IOException {
         final ByteArrayOutputStream os = new ByteArrayOutputStream();
         final byte[] buffer = new byte[1024];

distribution/tools/keystore-cli/src/test/java/org/opensearch/tools/cli/keystore/KeyStoreWrapperTests.java

@@ -60,7 +60,7 @@
 
 final class PemUtils {
 
-    private static final String BCFIPS = "BCFIPS";
+    public static final String BCFIPS = "BCFIPS";
 
     PemUtils() {
         throw new IllegalStateException("Utility class should not be instantiated");

libs/ssl-config/src/main/java/org/opensearch/common/ssl/PemUtils.java

@@ -63,7 +63,7 @@ dependencies {
   api "net.java.dev.jna:jna-platform:${versions.jna}"
   api 'com.microsoft.azure:msal4j:1.18.0'
   api 'com.nimbusds:oauth2-oidc-sdk:11.21'
-  api 'com.nimbusds:nimbus-jose-jwt:9.41.1'
+  api 'com.nimbusds:nimbus-jose-jwt:10.0.2'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,

plugins/repository-azure/build.gradle

@@ -0,0 +1 @@
+93347ea9247ae09e095575e10f9cae79c195fbb8

plugins/repository-azure/licenses/nimbus-jose-jwt-10.0.2.jar.sha1

@@ -48,8 +48,8 @@ opensearchplugin {
 }
 
 dependencies {
-  api 'com.google.api:api-common:1.8.1'
-  api 'com.google.api:gax:2.35.0'
+  api 'com.google.api:api-common:2.46.1'
+  api 'com.google.api:gax:2.63.1'
   api 'com.google.api:gax-httpjson:2.42.0'
 
   api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0'
@@ -202,9 +202,26 @@ thirdPartyAudit {
     'javax.jms.Message',
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener',
-    // Bump for gax 2.42.0
-    'com.google.api.gax.rpc.EndpointContext',
-    'com.google.api.gax.rpc.RequestMutator'
+
+    // opentelemetry-api is an optional dependency of com.google.api:gax
+    'io.opentelemetry.api.OpenTelemetry',
+    'io.opentelemetry.api.common.Attributes',
+    'io.opentelemetry.api.common.AttributesBuilder',
+    'io.opentelemetry.api.metrics.DoubleHistogram',
+    'io.opentelemetry.api.metrics.DoubleHistogramBuilder',
+    'io.opentelemetry.api.metrics.LongCounter',
+    'io.opentelemetry.api.metrics.LongCounterBuilder',
+    'io.opentelemetry.api.metrics.Meter',
+    'io.opentelemetry.api.metrics.MeterBuilder',
+
+    // slf4j is an optional dependency of com.google.api:gax
+    'org.slf4j.ILoggerFactory',
+    'org.slf4j.Logger',
+    'org.slf4j.LoggerFactory',
+    'org.slf4j.MDC',
+    'org.slf4j.event.Level',
+    'org.slf4j.helpers.NOPLogger',
+    'org.slf4j.spi.LoggingEventBuilder'
   )
 }
 

plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1

@@ -0,0 +1 @@
+b38a684c734963a72c204aa208dd31018d79bf3a

plugins/repository-gcs/build.gradle

@@ -0,0 +1 @@
+6c9a340608a63e24dc8acd8da84afd8ffecca4b7

plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1

@@ -10,6 +10,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.blobstore.BlobPath;
@@ -42,6 +43,7 @@
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+@LuceneTestCase.AwaitsFix(bugUrl = "Flakiness seen for this class")
 public class S3RemoteStoreIT extends RemoteStoreCoreTestCase {
 
     @Override

plugins/repository-gcs/licenses/api-common-2.46.1.jar.sha1

@@ -116,7 +116,8 @@ public void testReadWritePaths() throws IOException, InterruptedException {
             assertTrue("Path should exist: " + path, checkPathExists(path));
             assertTrue("Path should be readable: " + path, checkPathReadable(path));
             assertTrue("Path should be writable: " + path, checkPathWritable(path));
-            assertEquals("Path should be owned by opensearch:opensearch", "opensearch:opensearch", getPathOwnership(path));
+            String ownership = getPathOwnership(path);
+            assertTrue("Path should be owned by opensearch:opensearch or opensearch:adm", ownership.equals("opensearch:opensearch") || ownership.equals("opensearch:adm"));
         }
     }
 

plugins/repository-gcs/licenses/gax-2.35.0.jar.sha1

@@ -21,31 +21,21 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
-import static org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class SearchReplicaFilteringAllocationIT extends RemoteStoreBaseIntegTestCase {
+public class SearchReplicaAllocationIT extends RemoteStoreBaseIntegTestCase {
 
     @Override
     protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build();
     }
 
-    public void testSearchReplicaDedicatedIncludes() {
-        List<String> nodesIds = internalCluster().startNodes(3);
-        final String node_0 = nodesIds.get(0);
-        final String node_1 = nodesIds.get(1);
-        final String node_2 = nodesIds.get(2);
-        assertEquals(3, cluster().size());
+    public void testSearchReplicaAllocatedToDedicatedSearchNode() {
+        internalCluster().startClusterManagerOnlyNode();
+        String primaryNode = internalCluster().startDataOnlyNode();
+        internalCluster().startSearchOnlyNode();
 
-        client().admin()
-            .cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(
-                Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node_1 + "," + node_0)
-            )
-            .execute()
-            .actionGet();
+        assertEquals(3, cluster().size());
 
         createIndex(
             "test",
@@ -57,42 +47,16 @@ public void testSearchReplicaDedicatedIncludes() {
                 .build()
         );
         ensureGreen("test");
-        // ensure primary is not on node 0 or 1,
+        // ensure primary is not on searchNode
         IndexShardRoutingTable routingTable = getRoutingTable();
-        assertEquals(node_2, getNodeName(routingTable.primaryShard().currentNodeId()));
-
-        String existingSearchReplicaNode = getNodeName(routingTable.searchOnlyReplicas().get(0).currentNodeId());
-        String emptyAllowedNode = existingSearchReplicaNode.equals(node_0) ? node_1 : node_0;
-
-        // set the included nodes to the other open node, search replica should relocate to that node.
-        client().admin()
-            .cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", emptyAllowedNode))
-            .execute()
-            .actionGet();
-        ensureGreen("test");
-
-        routingTable = getRoutingTable();
-        assertEquals(node_2, getNodeName(routingTable.primaryShard().currentNodeId()));
-        assertEquals(emptyAllowedNode, getNodeName(routingTable.searchOnlyReplicas().get(0).currentNodeId()));
+        assertEquals(primaryNode, getNodeName(routingTable.primaryShard().currentNodeId()));
     }
 
     public void testSearchReplicaDedicatedIncludes_DoNotAssignToOtherNodes() {
-        List<String> nodesIds = internalCluster().startNodes(3);
-        final String node_0 = nodesIds.get(0);
-        final String node_1 = nodesIds.get(1);
-        final String node_2 = nodesIds.get(2);
+        internalCluster().startNodes(2);
+        final String node_1 = internalCluster().startSearchOnlyNode();
         assertEquals(3, cluster().size());
 
-        // set filter on 1 node and set search replica count to 2 - should leave 1 unassigned
-        client().admin()
-            .cluster()
-            .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node_1))
-            .execute()
-            .actionGet();
-
         logger.info("--> creating an index with no replicas");
         createIndex(
             "test",
@@ -115,9 +79,32 @@ public void testSearchReplicaDedicatedIncludes_DoNotAssignToOtherNodes() {
         assertEquals(1, routingTable.searchOnlyReplicas().stream().filter(ShardRouting::unassigned).count());
     }
 
+    public void testSearchReplicaDedicatedIncludes_WhenNotSetDoNotAssign() {
+        internalCluster().startNodes(2);
+        assertEquals(2, cluster().size());
+
+        createIndex(
+            "test",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1)
+                .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+                .build()
+        );
+        ensureYellowAndNoInitializingShards("test");
+        IndexShardRoutingTable routingTable = getRoutingTable();
+        assertNull(routingTable.searchOnlyReplicas().get(0).currentNodeId());
+
+        // Add a search node
+        final String searchNode = internalCluster().startSearchOnlyNode();
+
+        ensureGreen("test");
+        assertEquals(searchNode, getNodeName(getRoutingTable().searchOnlyReplicas().get(0).currentNodeId()));
+    }
+
     private IndexShardRoutingTable getRoutingTable() {
-        IndexShardRoutingTable routingTable = getClusterState().routingTable().index("test").getShards().get(0);
-        return routingTable;
+        return getClusterState().routingTable().index("test").getShards().get(0);
     }
 
     private String getNodeName(String id) {