Skip to content

Commit bc6a3ca

Browse files
original-brownbearoriginal-brownbear
committed
Merge remote-tracking branch 'elastic/main' into 96302
2 parents 43c6c20 + a6d715f commit bc6a3ca

File tree

15,456 files changed

+823833
-225096
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

15,456 files changed

+823833
-225096
lines changed

‎.backportrc.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"upstream" : "elastic/elasticsearch",
3-
"targetBranchChoices" : [ "main", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
3+
"targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
44
"targetPRLabels" : [ "backport" ],
55
"branchLabelMapping" : {
6-
"^v8.11.0$" : "main",
6+
"^v8.15.0$" : "main",
77
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
88
}
99
}

‎.buildkite/.editorconfig

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[*.ts]
2+
max_line_length = 120

‎.buildkite/.gitignore

+169
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
2+
3+
# Logs
4+
5+
logs
6+
_.log
7+
npm-debug.log_
8+
yarn-debug.log*
9+
yarn-error.log*
10+
lerna-debug.log*
11+
.pnpm-debug.log*
12+
13+
# Diagnostic reports (https://nodejs.org/api/report.html)
14+
15+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
16+
17+
# Runtime data
18+
19+
pids
20+
_.pid
21+
_.seed
22+
\*.pid.lock
23+
24+
# Directory for instrumented libs generated by jscoverage/JSCover
25+
26+
lib-cov
27+
28+
# Coverage directory used by tools like istanbul
29+
30+
coverage
31+
\*.lcov
32+
33+
# nyc test coverage
34+
35+
.nyc_output
36+
37+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
38+
39+
.grunt
40+
41+
# Bower dependency directory (https://bower.io/)
42+
43+
bower_components
44+
45+
# node-waf configuration
46+
47+
.lock-wscript
48+
49+
# Compiled binary addons (https://nodejs.org/api/addons.html)
50+
51+
build/Release
52+
53+
# Dependency directories
54+
55+
node_modules/
56+
jspm_packages/
57+
58+
# Snowpack dependency directory (https://snowpack.dev/)
59+
60+
web_modules/
61+
62+
# TypeScript cache
63+
64+
\*.tsbuildinfo
65+
66+
# Optional npm cache directory
67+
68+
.npm
69+
70+
# Optional eslint cache
71+
72+
.eslintcache
73+
74+
# Optional stylelint cache
75+
76+
.stylelintcache
77+
78+
# Microbundle cache
79+
80+
.rpt2_cache/
81+
.rts2_cache_cjs/
82+
.rts2_cache_es/
83+
.rts2_cache_umd/
84+
85+
# Optional REPL history
86+
87+
.node_repl_history
88+
89+
# Output of 'npm pack'
90+
91+
\*.tgz
92+
93+
# Yarn Integrity file
94+
95+
.yarn-integrity
96+
97+
# dotenv environment variable files
98+
99+
.env
100+
.env.development.local
101+
.env.test.local
102+
.env.production.local
103+
.env.local
104+
105+
# parcel-bundler cache (https://parceljs.org/)
106+
107+
.cache
108+
.parcel-cache
109+
110+
# Next.js build output
111+
112+
.next
113+
out
114+
115+
# Nuxt.js build / generate output
116+
117+
.nuxt
118+
dist
119+
120+
# Gatsby files
121+
122+
.cache/
123+
124+
# Comment in the public line in if your project uses Gatsby and not Next.js
125+
126+
# https://nextjs.org/blog/next-9-1#public-directory-support
127+
128+
# public
129+
130+
# vuepress build output
131+
132+
.vuepress/dist
133+
134+
# vuepress v2.x temp and cache directory
135+
136+
.temp
137+
.cache
138+
139+
# Docusaurus cache and generated files
140+
141+
.docusaurus
142+
143+
# Serverless directories
144+
145+
.serverless/
146+
147+
# FuseBox cache
148+
149+
.fusebox/
150+
151+
# DynamoDB Local files
152+
153+
.dynamodb/
154+
155+
# TernJS port file
156+
157+
.tern-port
158+
159+
# Stores VSCode versions used for testing VSCode extensions
160+
161+
.vscode-test
162+
163+
# yarn v2
164+
165+
.yarn/cache
166+
.yarn/unplugged
167+
.yarn/build-state.yml
168+
.yarn/install-state.gz
169+
.pnp.\*

‎.buildkite/README.md

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Elasticsearch CI Pipelines
2+
3+
This directory contains pipeline definitions and scripts for running Elasticsearch CI on Buildkite.
4+
5+
## Directory Structure
6+
7+
- [pipelines](pipelines/) - pipeline definitions/yml
8+
- [scripts](scripts/) - scripts used by pipelines, inside steps
9+
- [hooks](hooks/) - [Buildkite hooks](https://buildkite.com/docs/agent/v3/hooks), where global env vars and secrets are set
10+
11+
## Pipeline Definitions
12+
13+
Pipelines are defined using YAML files residing in [pipelines](pipelines/). These are mostly static definitions that are used as-is, but there are a few dynamically-generated exceptions (see below).
14+
15+
### Dynamically Generated Pipelines
16+
17+
Pull request pipelines are generated dynamically based on labels, files changed, and other properties of pull requests.
18+
19+
Non-pull request pipelines that include BWC version matrices must also be generated whenever the [list of BWC versions](../.ci/bwcVersions) is updated.
20+
21+
#### Pull Request Pipelines
22+
23+
Pull request pipelines are generated dynamically at CI time based on numerous properties of the pull request. See [scripts/pull-request](scripts/pull-request) for details.
24+
25+
#### BWC Version Matrices
26+
27+
For pipelines that include BWC version matrices, you will see one or more template files (e.g. [periodic.template.yml](pipelines/periodic.template.yml)) and a corresponding generated file (e.g. [periodic.yml](pipelines/periodic.yml)). The generated file is the one that is actually used by Buildkite.
28+
29+
These files are updated by running:
30+
31+
```bash
32+
./gradlew updateCIBwcVersions
33+
```
34+
35+
This also runs automatically during release procedures.
36+
37+
You should always make changes to the template files, and run the above command to update the generated files.
38+
39+
## Node / TypeScript
40+
41+
Node (technically `bun`), TypeScript, and related files are currently used to generate pipelines for pull request CI. See [scripts/pull-request](scripts/pull-request) for details.

‎.buildkite/bun.lockb

1.92 KB
Binary file not shown.

‎.buildkite/check-es-serverless.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
steps:
2-
- trigger: elasticsearch-serverless-update-submodule
2+
- trigger: elasticsearch-serverless-validate-submodule
33
label: ":elasticsearch: Check elasticsearch changes against serverless"
44
build:
55
message: "Validate latest elasticsearch changes"

‎.buildkite/hooks/pre-command

+48-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/bu
4848
export BUILDKITE_API_TOKEN
4949

5050
if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
51-
data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)
51+
data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
5252

5353
AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
5454
export AWS_ACCESS_KEY_ID
@@ -58,3 +58,50 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
5858

5959
unset data
6060
fi
61+
62+
if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
63+
DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
64+
export DRA_VAULT_ROLE_ID_SECRET
65+
66+
DRA_VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
67+
export DRA_VAULT_SECRET_ID_SECRET
68+
69+
DRA_VAULT_ADDR=https://secrets.elastic.co:8200
70+
export DRA_VAULT_ADDR
71+
fi
72+
73+
source .buildkite/scripts/third-party-test-credentials.sh
74+
75+
if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
76+
SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
77+
export SNYK_TOKEN
78+
fi
79+
80+
if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
81+
DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
82+
export DOCKER_REGISTRY_USERNAME
83+
84+
DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
85+
export DOCKER_REGISTRY_PASSWORD
86+
fi
87+
88+
if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
89+
# Run in the background, while the job continues
90+
nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
91+
fi
92+
93+
# Initialize the build scan and gobld annotations with empty/open <details> tags
94+
# This ensures that they are collapsible when they get appended to
95+
if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
96+
cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
97+
<details>
98+
99+
<summary>Gradle build scan links</summary>
100+
EOF
101+
102+
cat << EOF | buildkite-agent annotate --context "ctx-gobld-metrics" --style "info"
103+
<details>
104+
105+
<summary>Agent information from gobld</summary>
106+
EOF
107+
fi

‎.buildkite/hooks/pre-command.bat

+2
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,6 @@ set JOB_BRANCH=%BUILDKITE_BRANCH%
1818
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
1919
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
2020

21+
bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
22+
2123
exit /b 0

‎.buildkite/package.json

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "buildkite-pipelines",
3+
"type": "module",
4+
"devDependencies": {
5+
"@types/node": "^20.6.0",
6+
"bun-types": "latest",
7+
"yaml": "^2.3.2"
8+
},
9+
"peerDependencies": {
10+
"typescript": "^5.0.0"
11+
}
12+
}

‎.buildkite/packer_cache.sh

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
4+
5+
branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch'))
6+
for branch in "${branches[@]}"; do
7+
echo "Resolving dependencies for ${branch} branch"
8+
rm -rf "checkout/$branch"
9+
git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch"
10+
11+
CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd)
12+
CI_DIR="$CHECKOUT_DIR/.ci"
13+
14+
if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
15+
## On ARM we use a different properties file for setting java home
16+
## Also, we don't bother attempting to resolve dependencies for the 6.8 branch
17+
source "$CI_DIR/java-versions-aarch64.properties"
18+
export JAVA16_HOME="$HOME/.java/jdk16"
19+
else
20+
source "$CI_DIR/java-versions.properties"
21+
## We are caching BWC versions too, need these so we can build those
22+
export JAVA8_HOME="$HOME/.java/java8"
23+
export JAVA11_HOME="$HOME/.java/java11"
24+
export JAVA12_HOME="$HOME/.java/openjdk12"
25+
export JAVA13_HOME="$HOME/.java/openjdk13"
26+
export JAVA14_HOME="$HOME/.java/openjdk14"
27+
export JAVA15_HOME="$HOME/.java/openjdk15"
28+
export JAVA16_HOME="$HOME/.java/openjdk16"
29+
fi
30+
31+
export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
32+
"checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
33+
rm -rf "checkout/${branch}"
34+
done

‎.buildkite/pipelines/dra-workflow.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
steps:
2+
- command: .buildkite/scripts/dra-workflow.sh
3+
env:
4+
USE_DRA_CREDENTIALS: "true"
5+
agents:
6+
provider: gcp
7+
image: family/elasticsearch-ubuntu-2204
8+
machineType: custom-32-98304
9+
buildDirectory: /dev/shm/bk
10+
- wait
11+
# The hadoop build depends on the ES artifact
12+
# So let's trigger the hadoop build any time we build a new staging artifact
13+
- trigger: elasticsearch-hadoop-dra-workflow
14+
async: true
15+
build:
16+
branch: "${BUILDKITE_BRANCH}"
17+
env:
18+
DRA_WORKFLOW: staging
19+
if: build.env('DRA_WORKFLOW') == 'staging'

‎.buildkite/pipelines/ecs-dynamic-template-tests.yml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
steps:
2+
- label: ecs-dynamic-templates-tests
3+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true :x-pack:plugin:stack:javaRestTest
4+
timeout_in_minutes: 420
5+
agents:
6+
provider: gcp
7+
image: family/elasticsearch-ubuntu-2004
8+
diskSizeGb: 350
9+
machineType: custom-32-98304
10+
notify:
11+
- slack: "#es-delivery"
12+
if: build.state == "failed"
13+
- slack: "#es-data-management"
14+
if: build.state == "failed"
15+
- email: "[email protected]"
16+
if: build.state == "failed"

0 commit comments

Comments
 (0)