Merge branch 'opensearch-project:main' into main

Author: mohit10011999

.github/workflows/build-and-test.yml

@@ -18,7 +18,7 @@ jobs:
   build-test-linux:
     strategy:
       matrix:
-        java: [21]
+        java: [21, 23]
 
     name: Build CCR Plugin on Linux using Container Image
     runs-on: ubuntu-latest

.github/workflows/bwc.yml

@@ -13,12 +13,15 @@ jobs:
     # Job name
     name: Run BWC Test
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21, 23]
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 21
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
-          java-version: 21
+          java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v4

.github/workflows/dco-check.yml

@@ -0,0 +1,32 @@
+name: Publish snapshots to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+jobs:
+  build-and-publish-snapshots:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
+          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
+          echo "::add-mask::$SONATYPE_USERNAME"
+          echo "::add-mask::$SONATYPE_PASSWORD"
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository

.github/workflows/maven-publish.yml

@@ -55,6 +55,9 @@ jobs:
     if: ${{ 'True' == needs.req.outputs.isSecurityPluginAvailable }}
     # Job name
     name: Build and Run Security tests
+    strategy:
+      matrix:
+        java: [21, 23]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -67,10 +70,10 @@ jobs:
       - name: Run start commands
         run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 21
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
-          java-version: 21
+          java-version: ${{ matrix.java }}
           distribution: 'temurin'
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
@@ -84,7 +87,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: logs
+          name: logs-build-${{ matrix.java }}
           path: |
             build/testclusters/integTest-*/logs/*
             build/testclusters/leaderCluster-*/logs/*
@@ -103,6 +106,9 @@ jobs:
     if: ${{ 'True' == needs.req.outputs.isKnnPluginAvailable }}
     # Job name
     name: Build and Run Knn tests
+    strategy:
+      matrix:
+        java: [21, 23]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -115,10 +121,10 @@ jobs:
       - name: Run start commands
         run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 21
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
-          java-version: 21
+          java-version: ${{ matrix.java }}
           distribution: 'temurin'
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
@@ -131,7 +137,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: logs
+          name: logs-knnbuild-${{ matrix.java }}
           path: |
             build/testclusters/integTest-*/logs/*
             build/testclusters/leaderCluster-*/logs/*

.github/workflows/security-knn-tests.yml

@@ -35,10 +35,11 @@ import org.opensearch.gradle.test.RestIntegTestTask
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-beta1-SNAPSHOT")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "beta1")
         // e.g. 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT
         version_tokens = opensearch_version.tokenize('-')
+        opensearch_version_no_qualifier = version_tokens[0]
         opensearch_build = version_tokens[0] + '.0'
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
@@ -59,7 +60,12 @@ buildscript {
         if (!isSnapshot) {
             security_plugin_version = opensearch_build.replace("-SNAPSHOT","")
         }
-        knn_plugin_version = security_plugin_version
+
+        opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
+        security_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
+        knn_plugin_path = "build/dependencies/knn"
+        knn_plugin_download_url = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
+                '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-knn-' + security_no_snapshot + '.zip'
 
     }
 
@@ -241,21 +247,20 @@ def securityPluginFile = new Callable<RegularFile>() {
 
 def knnEnabled = findProperty("knn") == "true"
 
-if(knnEnabled == true ){
-    dependencies{
-        opensearchPlugin "org.opensearch.plugin:opensearch-knn:${knn_plugin_version}@zip"
-    }
-}
-
 def knnPluginFile = new Callable<RegularFile>() {
     @Override
     RegularFile call() throws Exception {
         return new RegularFile() {
             @Override
             File getAsFile() {
-                    return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts
-                            .find { ResolvedArtifact f -> f.name.contains('opensearch-knn') }
-                            .file
+                if (new File("$project.rootDir/$knn_plugin_path").exists()) {
+                    project.delete(files("$project.rootDir/$knn_plugin_path"))
+                }
+                project.mkdir knn_plugin_path
+                ant.get(src: knn_plugin_download_url,
+                        dest: knn_plugin_path,
+                        httpusecaches: false)
+                return fileTree(knn_plugin_path).getSingleFile()
             }
         }
     }
@@ -382,6 +387,8 @@ testClusters {
         if(knnEnabled) {
             plugin(provider(knnPluginFile))
             testDistribution = "ARCHIVE"
+            systemProperty "java.library.path", "${buildDir}/testclusters/leaderCluster-0/distro/${opensearch_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+            environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/leaderCluster-0/distro/${opensearch_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
         }
         int debugPort = 5005
 
@@ -399,6 +406,8 @@ testClusters {
         if(knnEnabled) {
             plugin(provider(knnPluginFile))
             testDistribution = "ARCHIVE"
+            systemProperty "java.library.path", "${buildDir}/testclusters/followCluster-0/distro/${opensearch_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+            environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/followCluster-0/distro/${opensearch_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
         }
         int debugPort = 5010
         if (_numNodes > 1) numberOfNodes = _numNodes
@@ -980,6 +989,16 @@ publishing {
             }
         }
     }
+    repositories {
+        maven {
+            name = "Snapshots"
+            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
+            credentials {
+                username "$System.env.SONATYPE_USERNAME"
+                password "$System.env.SONATYPE_PASSWORD"
+            }
+        }
+    }
 }
 
 // updateVersion: Task to auto increment to the next development iteration

build.gradle

@@ -1,16 +1,8 @@
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-#
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-#
-
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists