Merge remote-tracking branch 'upstream/main' into cache-impl-size-setting-bugfix

Signed-off-by: Peter Alfonsi <[email protected]>

Author: Peter Alfonsi

.github/workflows/detect-breaking-change.yml

@@ -17,7 +17,7 @@ jobs:
       with:
         cache-disabled: true
         arguments: japicmp
-        gradle-version: 8.7
+        gradle-version: 8.11
         build-root-directory: server
     - if: failure()
       run: cat server/build/reports/java-compatibility/report.txt

.github/workflows/gradle-check.yml

@@ -115,7 +115,7 @@ jobs:
 
       - name: Upload Coverage Report
         if: success()
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./codeCoverage.xml

CHANGELOG.md

@@ -20,25 +20,41 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
 - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271))
 - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
+- Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
+- Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
+- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
+- Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
 - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520))
-- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521))
-- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548))
+- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.28.0 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521), [#16808](https://github.com/opensearch-project/OpenSearch/pull/16808))
+- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241105-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548), [#16613](https://github.com/opensearch-project/OpenSearch/pull/16613))
 - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501))
 - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550))
-- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612))
-- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.3.0 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612), [#16854](https://github.com/opensearch-project/OpenSearch/pull/16854))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.47 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611), [#16807](https://github.com/opensearch-project/OpenSearch/pull/16807))
 - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610))
 - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614))
 - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655))
 - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661))
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665))
+- Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.3 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718), [#16858](https://github.com/opensearch-project/OpenSearch/pull/16858))
+- Bump `jackson` from 2.17.2 to 2.18.2 ([#16733](https://github.com/opensearch-project/OpenSearch/pull/16733))
+- Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716))
+- Bump `com.azure:azure-identity` from 1.13.2 to 1.14.2 ([#16778](https://github.com/opensearch-project/OpenSearch/pull/16778))
+- Bump Apache Lucene from 9.12.0 to 9.12.1 ([#16846](https://github.com/opensearch-project/OpenSearch/pull/16846))
+- Bump `com.gradle.develocity` from 3.18.2 to 3.19 ([#16855](https://github.com/opensearch-project/OpenSearch/pull/16855))
+- Bump `org.jline:jline` from 3.27.1 to 3.28.0 ([#16857](https://github.com/opensearch-project/OpenSearch/pull/16857))
+- Bump `com.azure:azure-core` from 1.51.0 to 1.54.1 ([#16856](https://github.com/opensearch-project/OpenSearch/pull/16856))
 
 ### Changed
+- Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
+- Make entries for dependencies from server/build.gradle to gradle version catalog ([#16707](https://github.com/opensearch-project/OpenSearch/pull/16707))
 
 ### Deprecated
+- Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))
 
 ### Removed
 
@@ -51,6 +67,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
 - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544))
 - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))
+- Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670))
+- [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560))
+- Bound the size of cache in deprecation logger ([16702](https://github.com/opensearch-project/OpenSearch/issues/16702))
+- Ensure consistency of system flag on IndexMetadata after diff is applied ([#16644](https://github.com/opensearch-project/OpenSearch/pull/16644))
+- Skip remote-repositories validations for node-joins when RepositoriesService is not in sync with cluster-state ([#16763](https://github.com/opensearch-project/OpenSearch/pull/16763))
+- Fix _list/shards API failing when closed indices are present ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606))
+- Fix remote shards balance ([#15335](https://github.com/opensearch-project/OpenSearch/pull/15335))
 - Fix max request cache size settings not working properly with pluggable caching ([#16636](https://github.com/opensearch-project/OpenSearch/pull/16636))
 
 ### Security

build.gradle

@@ -162,14 +162,20 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
   return [key, value]
 }
 
-    /**
-     * Using 'git' command line (if available), tries to fetch the commit date of the current revision
-     * @return commit date of the current revision or 0 if it is not available
-     */
+// See please https://docs.gradle.org/8.11/userguide/service_injection.html#execoperations
+interface InjectedExecOps {
+  @Inject ExecOperations getExecOps()
+}
+
+/**
+ * Using 'git' command line (if available), tries to fetch the commit date of the current revision
+ * @return commit date of the current revision or 0 if it is not available
+ */
 long gitRevisionDate = {
+  def execOps = project.objects.newInstance(InjectedExecOps)
   // Try to get last commit date as Unix timestamp
   try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) {
-     ExecResult result = project.exec(spec -> {
+     ExecResult result = execOps.execOps.exec(spec -> {
         spec.setIgnoreExitValue(true);
         spec.setStandardOutput(stdout);
         spec.commandLine("git", "log", "-1", "--format=%ct");
@@ -362,7 +368,7 @@ allprojects {
         if ((dep instanceof ProjectDependency) == false) {
           return
         }
-        Project upstreamProject = dep.dependencyProject
+        Project upstreamProject = project.project(dep.path)
         if (upstreamProject == null) {
           return
         }
@@ -438,7 +444,7 @@ gradle.projectsEvaluated {
 
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
-        Project upstreamProject = dep.dependencyProject
+        Project upstreamProject = project.project(dep.path)
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/OptionalDependenciesPlugin.groovy

@@ -79,7 +79,7 @@ class OptionalDependenciesPlugin implements Plugin<Project> {
 
                                 if (foundDep) {
                                     if (foundDep.optional) {
-                                        foundDep.optional.value = 'true'
+                                        foundDep.optional*.value = 'true'
                                     } else {
                                         foundDep.appendNode(OPTIONAL_IDENTIFIER, 'true')
                                     }

buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy

@@ -56,9 +56,12 @@ class TestWithDependenciesPlugin implements Plugin<Project> {
 
         project.configurations.testImplementation.dependencies.all { Dependency dep ->
             // this closure is run every time a compile dependency is added
-            if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
-                project.gradle.projectsEvaluated {
-                    addPluginResources(project, dep.dependencyProject)
+            if (dep instanceof ProjectDependency) {
+                Project dependencyProject = project.project(((ProjectDependency)dep).path)
+                if (dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
+                    project.gradle.projectsEvaluated {
+                        addPluginResources(project, dependencyProject)
+                    }
                 }
             }
         }

buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java

@@ -71,6 +71,11 @@ public class LoggedExec extends Exec implements FileSystemOperationsAware {
     private Consumer<Logger> outputLogger;
     private FileSystemOperations fileSystemOperations;
 
+    interface InjectedExecOps {
+        @Inject
+        ExecOperations getExecOps();
+    }
+
     @Inject
     public LoggedExec(FileSystemOperations fileSystemOperations) {
         this.fileSystemOperations = fileSystemOperations;
@@ -133,15 +138,17 @@ public void setSpoolOutput(boolean spoolOutput) {
     }
 
     public static ExecResult exec(Project project, Action<ExecSpec> action) {
-        return genericExec(project::exec, action);
+        final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class);
+        return exec(execOps.getExecOps(), action);
     }
 
     public static ExecResult exec(ExecOperations execOperations, Action<ExecSpec> action) {
         return genericExec(execOperations::exec, action);
     }
 
     public static ExecResult javaexec(Project project, Action<JavaExecSpec> action) {
-        return genericExec(project::javaexec, action);
+        final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class);
+        return genericExec(execOps.getExecOps()::javaexec, action);
     }
 
     /** Returns JVM arguments suitable for a short-lived forked task */

buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java

@@ -121,7 +121,7 @@ public String call() throws Exception {
                         Node dependencyNode = dependenciesNode.appendNode("dependency");
                         dependencyNode.appendNode("groupId", dependency.getGroup());
                         ProjectDependency projectDependency = (ProjectDependency) dependency;
-                        String artifactId = getArchivesBaseName(projectDependency.getDependencyProject());
+                        String artifactId = getArchivesBaseName(project.project(projectDependency.getPath()));
                         dependencyNode.appendNode("artifactId", artifactId);
                         dependencyNode.appendNode("version", dependency.getVersion());
                         dependencyNode.appendNode("scope", "compile");