Merge branch 'main' into json-parser-exc

Author: smalyshev

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -303,8 +303,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
+      - label: "{{matrix.image}} / 8.17.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,10 +317,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.6
+          BWC_VERSION: 8.17.7
 
-      - label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
+      - label: "{{matrix.image}} / 8.18.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -333,7 +333,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.1
+          BWC_VERSION: 8.18.2
 
       - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
+      - label: "{{matrix.image}} / 9.0.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.1
+          BWC_VERSION: 9.0.2
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.template.yml

@@ -78,8 +78,8 @@ steps:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
-      - label: "{{matrix.ES_RUNTIME_JAVA}} / entitlements={{matrix.ENTITLEMENTS_ENABLED}} / {{matrix.GRADLE_TASK}} / java-matrix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline="-Des.entitlements.enabled=$$ENTITLEMENTS_ENABLED" $$GRADLE_TASK
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -94,9 +94,6 @@ steps:
               - checkPart4
               - checkPart5
               - checkRestCompat
-            ENTITLEMENTS_ENABLED:
-              - "true"
-              - "false"
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -105,7 +102,6 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-          ENTITLEMENTS_ENABLED: "{{matrix.ENTITLEMENTS_ENABLED}}"
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.6#bwcTest
+      - label: 8.17.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.6
+          BWC_VERSION: 8.17.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
+      - label: 8.18.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.1
+          BWC_VERSION: 8.18.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
+      - label: 9.0.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.1
+          BWC_VERSION: 9.0.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -497,8 +497,8 @@ steps:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
-      - label: "{{matrix.ES_RUNTIME_JAVA}} / entitlements={{matrix.ENTITLEMENTS_ENABLED}} / {{matrix.GRADLE_TASK}} / java-matrix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline="-Des.entitlements.enabled=$$ENTITLEMENTS_ENABLED" $$GRADLE_TASK
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -513,9 +513,6 @@ steps:
               - checkPart4
               - checkPart5
               - checkRestCompat
-            ENTITLEMENTS_ENABLED:
-              - "true"
-              - "false"
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -524,7 +521,6 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-          ENTITLEMENTS_ENABLED: "{{matrix.ENTITLEMENTS_ENABLED}}"
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300
@@ -533,7 +529,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/part-1-entitlements.yml

@@ -2,10 +2,34 @@
 
 set -euo pipefail
 
-echo "steps:"
-
 source .buildkite/scripts/branches.sh
 
+# We use that filtering to keep different schedule for different branches
+if [ -n "${INCLUDED_BRANCHES:-}" ]; then
+  # If set, only trigger the pipeline for the specified branches
+  IFS=',' read -r -a BRANCHES <<< "${INCLUDED_BRANCHES}"
+elif [ -n "${EXCLUDED_BRANCHES:-}" ]; then
+  # If set, listed branches will be excluded from the list of branches in branches.json
+  IFS=',' read -r -a EXCLUDED_BRANCHES <<< "${EXCLUDED_BRANCHES}"
+  FILTERED_BRANCHES=()
+    for BRANCH in "${BRANCHES[@]}"; do
+      EXCLUDE=false
+      for EXCLUDED_BRANCH in "${EXCLUDED_BRANCHES[@]}"; do
+        if [ "$BRANCH" == "$EXCLUDED_BRANCH" ]; then
+          EXCLUDE=true
+          break
+        fi
+      done
+      if [ "$EXCLUDE" = false ]; then
+        FILTERED_BRANCHES+=("$BRANCH")
+      fi
+    done
+    BRANCHES=("${FILTERED_BRANCHES[@]}")
+fi
+
+
+echo "steps:"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')

.buildkite/pipelines/pull-request/part-2-entitlements.yml

@@ -7,6 +7,7 @@ set -euo pipefail
 # The second/lowercase export is what the tests expect/require
 
 if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
@@ -19,6 +20,7 @@ if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)

.buildkite/pipelines/pull-request/part-3-entitlements.yml

@@ -16,8 +16,8 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.6"
-  - "8.18.1"
+  - "8.17.7"
+  - "8.18.2"
   - "8.19.0"
-  - "9.0.1"
+  - "9.0.2"
   - "9.1.0"

.buildkite/pipelines/pull-request/part-4-entitlements.yml

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.17.6"
-  - "8.18.1"
+  - "8.17.7"
+  - "8.18.2"
   - "8.19.0"
-  - "9.0.1"
+  - "9.0.2"
   - "9.1.0"

.buildkite/pipelines/pull-request/part-5-entitlements.yml

@@ -82,7 +82,7 @@ In case of updating a dependency, ensure to remove the unused entry of the outda
 
 You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
 ```
->./gradlew --write-verification-metadata sha256 precommit
+./gradlew --write-verification-metadata sha256 precommit
 ```
 
 The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
@@ -92,10 +92,10 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
->A manual confirmation of the Gradle generated checksums is currently not mandatory.
->If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
->Please replace the content of the `origin` attribute by `official site` in that case.
->
+> [!Tip]  
+> A manual confirmation of the Gradle generated checksums is currently not mandatory.
+> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
+> Please replace the content of the `origin` attribute by `official site` in that case.
 
 #### Custom plugin and task implementations
 
@@ -144,7 +144,7 @@ To wire this registered cluster into a `TestClusterAware` task (e.g. `RestIntegT
 Additional integration tests for a certain Elasticsearch modules that are specific to certain cluster configuration can be declared in a separate so called `qa` subproject of your module.
 
 The benefit of a dedicated project for these tests are:
-- `qa` projects are dedicated two specific use-cases and easier to maintain
+- `qa` projects are dedicated to specific use-cases and easier to maintain
 - It keeps the specific test logic separated from the common test logic.
 - You can run those tests in parallel to other projects of the build.
 
@@ -229,13 +229,9 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
----
-
-**NOTE**
-
-You should only use that approach locally or on a developer branch for production dependencies as we do
+> [!Note]  
+> You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
----
 
 #### How to use a custom third party artifact?
 
@@ -265,12 +261,9 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
----
-**NOTE**
-
-As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+> [!Note]  
+> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
-For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
+> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
 that supports meta-data, it will use the second repository to provide the module.
-Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)
----
+> Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)