Merge remote-tracking branch 'origin/main' into star-ff

Signed-off-by: Rishab Nahata <[email protected]>

Author: imRishN

.github/CODEOWNERS

@@ -1,37 +1,31 @@
-# CODEOWNERS manages notifications, not PR approvals
-# For PR approvals see /.github/workflows/maintainer-approval.yml
-
-# Files have a single rule applied, the last match decides the owner
-# If you would like to more specifically apply ownership, include existing owner in new sub fields
-
 # To verify changes of CODEOWNERS file
 # In VSCode
 #   1. Install extension https://marketplace.visualstudio.com/items?itemName=jasonnutter.vscode-codeowners
 #   2. Go to a file
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @anasalkouz @andrross @ashking94 @bugmakerrrrrr @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jainankitk @kotwanikunal @linuxpi @mch2 @msfroh @nknize @owaiskazi19  @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+* @opensearch-project/opensearch-core-maintainers
 
-/modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/modules/transport-netty4/ @peternied
+/modules/lang-painless/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/parent-join/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/transport-netty4/ @opensearch-project/opensearch-core-maintainers @peternied
 
-/plugins/identity-shiro/ @peternied @cwperks
+/plugins/identity-shiro/ @opensearch-project/opensearch-core-maintainers @peternied @cwperks
 
-/server/src/internalClusterTest/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/internalClusterTest/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
-/server/src/main/java/org/opensearch/extensions/ @peternied
-/server/src/main/java/org/opensearch/identity/ @peternied @cwperks 
-/server/src/main/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/main/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/main/java/org/opensearch/threadpool/ @jed326 @peternied
-/server/src/main/java/org/opensearch/transport/ @peternied
+/server/src/main/java/org/opensearch/extensions/ @opensearch-project/opensearch-core-maintainers @peternied
+/server/src/main/java/org/opensearch/identity/ @opensearch-project/opensearch-core-maintainers @peternied @cwperks 
+/server/src/main/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/threadpool/ @opensearch-project/opensearch-core-maintainers @jed326 @peternied
+/server/src/main/java/org/opensearch/transport/ @opensearch-project/opensearch-core-maintainers @peternied
 
-/server/src/test/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/test/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
-/.github/ @jed326 @peternied
+/.github/ @opensearch-project/opensearch-core-maintainers @jed326 @peternied
 
-/MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/MAINTAINERS.md @opensearch-project/opensearch-core-maintainers

.github/dependabot.yml

@@ -1403,14 +1403,6 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
-  - directory: /test/fixtures/old-elasticsearch/
-    open-pull-requests-limit: 1
-    package-ecosystem: gradle
-    schedule:
-      interval: weekly
-    labels:
-      - "dependabot"
-      - "dependencies"
   - directory: /test/fixtures/s3-fixture/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/workflows/maintainer-approval.yml

@@ -6,37 +6,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 3.x]
 ### Added
 - [Rule based auto-tagging] Add get rule API ([#17336](https://github.com/opensearch-project/OpenSearch/pull/17336))
-- Add multi-threaded writer support in pull-based ingestion ([#17912](https://github.com/opensearch-project/OpenSearch/pull/17912))
-- Unset discovery nodes for every transport node actions request ([#17682](https://github.com/opensearch-project/OpenSearch/pull/17682))
 - Implement parallel shard refresh behind cluster settings ([#17782](https://github.com/opensearch-project/OpenSearch/pull/17782))
 - Bump OpenSearch Core main branch to 3.0.0 ([#18039](https://github.com/opensearch-project/OpenSearch/pull/18039))
-- [Star Tree] Support of Boolean Queries in Aggregations ([#17941](https://github.com/opensearch-project/OpenSearch/pull/17941))
 - Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/))
-- Enabled default throttling for all tasks submitted to cluster manager ([#17711](https://github.com/opensearch-project/OpenSearch/pull/17711))
 - Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988))
+- Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088))
+- Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139))
 
 ### Changed
-- Change the default max header size from 8KB to 16KB. ([#18024](https://github.com/opensearch-project/OpenSearch/pull/18024))
-- Avoid invalid retries in multiple replicas when querying [#17370](https://github.com/opensearch-project/OpenSearch/pull/17370)
-* Enable concurrent_segment_search auto mode by default[#17978](https://github.com/opensearch-project/OpenSearch/pull/17978)
-- Skip approximation when `track_total_hits` is set to `true` [#18017](https://github.com/opensearch-project/OpenSearch/pull/18017)
 - [Star tree] Removing star tree indexing and queries added till 2.19 out of expermental [#18070](https://github.com/opensearch-project/OpenSearch/pull/18070)
 
 ### Dependencies
 - Bump `com.google.code.gson:gson` from 2.12.1 to 2.13.0 ([#17923](https://github.com/opensearch-project/OpenSearch/pull/17923))
 - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.0 to 4.9.3 ([#17922](https://github.com/opensearch-project/OpenSearch/pull/17922))
 - Bump `com.microsoft.azure:msal4j` from 1.18.0 to 1.20.0 ([#17925](https://github.com/opensearch-project/OpenSearch/pull/17925))
+- Update Apache HttpClient5 and HttpCore5 (CVE-2025-27820) ([#18152](https://github.com/opensearch-project/OpenSearch/pull/18152))
+- Bump `org.apache.commons:commons-collections4` from 4.4 to 4.5.0 ([#18101](https://github.com/opensearch-project/OpenSearch/pull/18101))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
-- Fix ingest pipeline cannot be executed when retry the failed index requests for update_by_query API and reindex API ([#18003](https://github.com/opensearch-project/OpenSearch/pull/18003))
-- With creation of FilterFieldType, we need unwrap all the MappedFieldType before using the instanceof check. ([#17951](https://github.com/opensearch-project/OpenSearch/pull/17951))
 - Fix simultaneously creating a snapshot and updating the repository can potentially trigger an infinite loop ([#17532](https://github.com/opensearch-project/OpenSearch/pull/17532))
 - Remove package org.opensearch.transport.grpc and replace with org.opensearch.plugin.transport.grpc ([#18031](https://github.com/opensearch-project/OpenSearch/pull/18031))
+- Fix the native plugin installation error cause by the pgp public key change ([#18147](https://github.com/opensearch-project/OpenSearch/pull/18147))
 
 ### Security
 
-[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/aa0e724e...main
+[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.0...main

CHANGELOG.md

@@ -551,7 +551,6 @@ subprojects {
         includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
         includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
         includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
-        includeClasses.add("org.opensearch.cluster.MinimumClusterManagerNodesIT")
       }
     }
   }

build.gradle

@@ -0,0 +1 @@
+29aafa2d5ced55ed75dab37cce5e125fb06e54d8

client/rest/licenses/httpclient5-5.4.1.jar.sha1

@@ -0,0 +1 @@
+ea47f0fe6e00ffb07cec3a0cb1bb801b1a9cc353

client/rest/licenses/httpclient5-5.4.4.jar.sha1

@@ -0,0 +1 @@
+3742a9a9ba3a5a0d45be230093b52a1302a561e2

client/rest/licenses/httpcore5-5.3.2.jar.sha1

@@ -0,0 +1 @@
+584f61333473c03458ccb38b7fa9a06b847b4046

client/rest/licenses/httpcore5-5.3.4.jar.sha1

@@ -382,9 +382,6 @@ public void testHeaders() throws Exception {
             if (method.equals("HEAD") == false) {
                 standardHeaders.add("Content-length");
             }
-            if (method.equals("HEAD") == true || method.equals("GET") == true || method.equals("OPTIONS") == true) {
-                standardHeaders.add("Upgrade");
-            }
 
             final Header[] requestHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header");
             final int statusCode = randomStatusCode(getRandom());

client/rest/licenses/httpcore5-h2-5.3.2.jar.sha1

@@ -0,0 +1 @@
+29aafa2d5ced55ed75dab37cce5e125fb06e54d8

client/rest/licenses/httpcore5-h2-5.3.4.jar.sha1

@@ -0,0 +1 @@
+ea47f0fe6e00ffb07cec3a0cb1bb801b1a9cc353

client/rest/licenses/httpcore5-reactive-5.3.2.jar.sha1

@@ -26,6 +26,9 @@ WorkingDirectory=/usr/share/opensearch
 User=opensearch
 Group=opensearch
 
+ExecStartPre=/bin/mkdir -p /dev/shm/performanceanalyzer
+ExecStartPre=/bin/chown opensearch:opensearch /dev/shm/performanceanalyzer
+
 ExecStart=/usr/share/opensearch/bin/systemd-entrypoint -p ${PID_DIR}/opensearch.pid --quiet
 
 # StandardOutput is configured to redirect to journalctl since
@@ -97,6 +100,8 @@ LockPersonality=yes
 # System call filterings which restricts which system calls a process can make
 # @ means allowed
 # ~ means not allowed
+# These syscalls are related to mmap which is needed for OpenSearch Services
+SystemCallFilter=madvise mincore mlock mlock2 munlock get_mempolicy sched_getaffinity sched_setaffinity fcntl
 SystemCallFilter=@system-service
 SystemCallFilter=~@reboot
 SystemCallFilter=~@swap
@@ -130,11 +135,12 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
 
 ReadWritePaths=/var/log/opensearch
 ReadWritePaths=/var/lib/opensearch
+ReadWritePaths=/dev/shm/
 ReadWritePaths=-/etc/opensearch
 ReadWritePaths=-/mnt/snapshots
 
 ## Allow read access to system files
-ReadOnlyPaths=/etc/os-release /usr/lib/os-release /etc/system-release
+ReadOnlyPaths=-/etc/os-release -/usr/lib/os-release -/etc/system-release
 
 ## Allow read access to Linux IO stats
 ReadOnlyPaths=/proc/self/mountinfo /proc/diskstats
@@ -149,17 +155,25 @@ RestrictNamespaces=true
 NoNewPrivileges=true
 
 # Memory and execution protection
-MemoryDenyWriteExecute=true           # Prevent creating writable executable memory mappings
-SystemCallArchitectures=native        # Allow only native system calls
-KeyringMode=private                   # Service does not share key material with other services
-LockPersonality=true                  # Prevent changing ABI personality
-RestrictSUIDSGID=true                 # Prevent creating SUID/SGID files
-RestrictRealtime=true                 # Prevent acquiring realtime scheduling
-ProtectHostname=true                  # Prevent changes to system hostname
-ProtectKernelLogs=true                # Prevent reading/writing kernel logs
-ProtectClock=true                     # Prevent tampering with the system clock
+
+# Allow only native system calls
+SystemCallArchitectures=native
+# Service does not share key material with other services
+KeyringMode=private
+# Prevent changing ABI personality
+LockPersonality=true
+# Prevent creating SUID/SGID files
+RestrictSUIDSGID=true
+# Prevent acquiring realtime scheduling
+RestrictRealtime=true
+# Prevent changes to system hostname
+ProtectHostname=true
+# Prevent reading/writing kernel logs
+ProtectKernelLogs=true
+# Prevent tampering with the system clock
+ProtectClock=true
 
 [Install]
 WantedBy=multi-user.target
 
-# Built for ${project.name}-${project.version} (${project.name})
+# Built for ${project.name}-${project.version} (${project.name})

client/rest/licenses/httpcore5-reactive-5.3.4.jar.sha1

@@ -40,6 +40,7 @@
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.Constants;
 import org.bouncycastle.bcpg.ArmoredInputStream;
+import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider;
 import org.bouncycastle.openpgp.PGPException;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.bouncycastle.openpgp.PGPPublicKeyRingCollection;
@@ -90,6 +91,7 @@
 import java.nio.file.attribute.PosixFilePermissions;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -632,6 +634,7 @@ void verifySignature(final Path zip, final String urlString) throws IOException,
             // compute the signature of the downloaded plugin zip
             final PGPPublicKeyRingCollection collection = new PGPPublicKeyRingCollection(ain, new JcaKeyFingerprintCalculator());
             final PGPPublicKey key = collection.getPublicKey(signature.getKeyID());
+            Security.addProvider(new BouncyCastleFipsProvider());
             signature.init(new JcaPGPContentVerifierBuilderProvider().setProvider("BCFIPS"), key);
             final byte[] buffer = new byte[1024];
             int read;