Skip to content

Commit 94bd169

Browse files
original-brownbearoriginal-brownbear
committed
Merge remote-tracking branch 'elastic/main' into some-counting-cleanup
2 parents 325ee1d + 4a8bc57 commit 94bd169

File tree

2,530 files changed

+57820
-23652
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,530 files changed

+57820
-23652
lines changed

‎.buildkite/pipelines/intake.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ steps:
5656
timeout_in_minutes: 300
5757
matrix:
5858
setup:
59-
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
59+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
6060
agents:
6161
provider: gcp
6262
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/periodic-packaging.yml

+12-12
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,8 @@ steps:
287287
env:
288288
BWC_VERSION: 8.15.5
289289

290-
- label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
291-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
290+
- label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
291+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
292292
timeout_in_minutes: 300
293293
matrix:
294294
setup:
@@ -301,10 +301,10 @@ steps:
301301
machineType: custom-16-32768
302302
buildDirectory: /dev/shm/bk
303303
env:
304-
BWC_VERSION: 8.16.7
304+
BWC_VERSION: 8.16.6
305305

306-
- label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade"
307-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5
306+
- label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
307+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
308308
timeout_in_minutes: 300
309309
matrix:
310310
setup:
@@ -317,10 +317,10 @@ steps:
317317
machineType: custom-16-32768
318318
buildDirectory: /dev/shm/bk
319319
env:
320-
BWC_VERSION: 8.17.5
320+
BWC_VERSION: 8.17.6
321321

322-
- label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
323-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
322+
- label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
323+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
324324
timeout_in_minutes: 300
325325
matrix:
326326
setup:
@@ -333,7 +333,7 @@ steps:
333333
machineType: custom-16-32768
334334
buildDirectory: /dev/shm/bk
335335
env:
336-
BWC_VERSION: 8.18.0
336+
BWC_VERSION: 8.18.1
337337

338338
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
339339
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -351,8 +351,8 @@ steps:
351351
env:
352352
BWC_VERSION: 8.19.0
353353

354-
- label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
355-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
354+
- label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
355+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
356356
timeout_in_minutes: 300
357357
matrix:
358358
setup:
@@ -365,7 +365,7 @@ steps:
365365
machineType: custom-16-32768
366366
buildDirectory: /dev/shm/bk
367367
env:
368-
BWC_VERSION: 9.0.0
368+
BWC_VERSION: 9.0.1
369369

370370
- label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
371371
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

‎.buildkite/pipelines/periodic.yml

+14-14
Original file line numberDiff line numberDiff line change
@@ -306,8 +306,8 @@ steps:
306306
- signal_reason: agent_stop
307307
limit: 3
308308

309-
- label: 8.16.7 / bwc
310-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
309+
- label: 8.16.6 / bwc
310+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
311311
timeout_in_minutes: 300
312312
agents:
313313
provider: gcp
@@ -316,7 +316,7 @@ steps:
316316
buildDirectory: /dev/shm/bk
317317
preemptible: true
318318
env:
319-
BWC_VERSION: 8.16.7
319+
BWC_VERSION: 8.16.6
320320
retry:
321321
automatic:
322322
- exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
325325
- signal_reason: agent_stop
326326
limit: 3
327327

328-
- label: 8.17.5 / bwc
329-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest
328+
- label: 8.17.6 / bwc
329+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.6#bwcTest
330330
timeout_in_minutes: 300
331331
agents:
332332
provider: gcp
@@ -335,7 +335,7 @@ steps:
335335
buildDirectory: /dev/shm/bk
336336
preemptible: true
337337
env:
338-
BWC_VERSION: 8.17.5
338+
BWC_VERSION: 8.17.6
339339
retry:
340340
automatic:
341341
- exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
344344
- signal_reason: agent_stop
345345
limit: 3
346346

347-
- label: 8.18.0 / bwc
348-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
347+
- label: 8.18.1 / bwc
348+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
349349
timeout_in_minutes: 300
350350
agents:
351351
provider: gcp
@@ -354,7 +354,7 @@ steps:
354354
buildDirectory: /dev/shm/bk
355355
preemptible: true
356356
env:
357-
BWC_VERSION: 8.18.0
357+
BWC_VERSION: 8.18.1
358358
retry:
359359
automatic:
360360
- exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
382382
- signal_reason: agent_stop
383383
limit: 3
384384

385-
- label: 9.0.0 / bwc
386-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
385+
- label: 9.0.1 / bwc
386+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
387387
timeout_in_minutes: 300
388388
agents:
389389
provider: gcp
@@ -392,7 +392,7 @@ steps:
392392
buildDirectory: /dev/shm/bk
393393
preemptible: true
394394
env:
395-
BWC_VERSION: 9.0.0
395+
BWC_VERSION: 9.0.1
396396
retry:
397397
automatic:
398398
- exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
486486
setup:
487487
ES_RUNTIME_JAVA:
488488
- openjdk21
489-
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
489+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
490490
agents:
491491
provider: gcp
492492
image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
533533
ES_RUNTIME_JAVA:
534534
- openjdk21
535535
- openjdk23
536-
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
536+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
537537
agents:
538538
provider: gcp
539539
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/pull-request/build-benchmark.yml

+8-5
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@ steps:
77
command: |
88
.ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
99
.ci/scripts/install-gradle-profiler.sh
10-
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-{{matrix.part}}.scenarios --project-dir . --output-dir profile-out
11-
mkdir build
12-
tar -czf build/$BUILDKITE_BUILD_NUMBER.tar.bz2 profile-out
10+
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-{{matrix.part}}.scenarios --measure-config-time --project-dir . --output-dir profile-out
11+
mkdir -p build
12+
buildkite-agent artifact upload "profile-out/*"
13+
cat << EOF | buildkite-agent annotate --style "info"
14+
Benchmark Html Report <a href="artifact://profile-out/benchmark.html">Benchmark Html Report</a>
15+
EOF
1316
matrix:
1417
setup:
1518
part:
16-
- part1
17-
- part2
19+
- esql-tests
1820
env:
1921
BUILD_PERFORMANCE_TEST: "true"
22+
2023
agents:
2124
provider: gcp
2225
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/pull-request/part-1-fips.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
config:
2-
allow-labels: "Team:Security"
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
35
steps:
46
- label: part-1-fips
57
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart1

‎.buildkite/pipelines/pull-request/part-2-fips.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
config:
2-
allow-labels: "Team:Security"
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
35
steps:
46
- label: part-2-fips
57
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart2

‎.buildkite/pipelines/pull-request/part-3-fips.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
config:
2-
allow-labels: "Team:Security"
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
35
steps:
46
- label: part-3-fips
57
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart3

‎.buildkite/pipelines/pull-request/part-4-fips.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
config:
2-
allow-labels: "Team:Security"
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
35
steps:
46
- label: part-4-fips
57
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4

‎.buildkite/pipelines/pull-request/part-5-fips.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
config:
2-
allow-labels: "Team:Security"
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
35
steps:
46
- label: part-5-fips
57
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart5

‎.buildkite/scripts/dra-workflow.sh

+1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ echo --- Building release artifacts
7070
$VERSION_QUALIFIER_ARG \
7171
buildReleaseArtifacts \
7272
exportCompressedDockerImages \
73+
exportDockerContexts \
7374
:distribution:generateDependenciesReport
7475

7576
PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

‎.buildkite/scripts/dra-workflow.trigger.sh

-4
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ echo "steps:"
77
source .buildkite/scripts/branches.sh
88

99
for BRANCH in "${BRANCHES[@]}"; do
10-
if [[ "$BRANCH" == "9.0" ]]; then
11-
continue
12-
fi
13-
1410
INTAKE_PIPELINE_SLUG="elasticsearch-intake"
1511
BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
1612
LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')

‎.ci/bwcVersions

+4-4
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ BWC_VERSION:
1515
- "8.13.4"
1616
- "8.14.3"
1717
- "8.15.5"
18-
- "8.16.7"
19-
- "8.17.5"
20-
- "8.18.0"
18+
- "8.16.6"
19+
- "8.17.6"
20+
- "8.18.1"
2121
- "8.19.0"
22-
- "9.0.0"
22+
- "9.0.1"
2323
- "9.1.0"

‎.ci/scripts/install-gradle-profiler.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44
# profiler version we wanna install
5-
PROFILER_VERSION="0.16.0"
6-
wget https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
5+
PROFILER_VERSION="0.22.0"
6+
wget https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
77
unzip $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
88
mv $WORKSPACE/gradle-profiler-$PROFILER_VERSION $WORKSPACE/gradle-profiler

‎.ci/scripts/packaging-test.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# opensuse 15 has a missing dep for systemd
44

55
if which zypper > /dev/null ; then
6-
sudo zypper install -y insserv-compat
6+
sudo zypper install -y insserv-compat docker-buildx
77
fi
88

99
if [ -e /etc/sysctl.d/99-gce.conf ]; then

‎.ci/snapshotBwcVersions

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
BWC_VERSION:
2-
- "8.16.7"
3-
- "8.17.5"
4-
- "8.18.0"
2+
- "8.17.6"
3+
- "8.18.1"
54
- "8.19.0"
6-
- "9.0.0"
5+
- "9.0.1"
76
- "9.1.0"

‎benchmarks/README.md

+31-5
Original file line numberDiff line numberDiff line change
@@ -82,19 +82,21 @@ To get realistic results, you should exercise care when running benchmarks. Here
8282
NOTE: Linux only. Sorry Mac and Windows.
8383

8484
Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and the JDK's `hsdis`.
85-
`perf` is generally available via `apg-get install perf` or `pacman -S perf`. `hsdis` you'll want to compile from source. is a little more involved. This worked
85+
`perf` is generally available via `apg-get install perf` or `pacman -S perf linux-tools`. `hsdis` you'll want to compile from source. is a little more involved. This worked
8686
on 2020-08-01:
8787

8888
```
8989
git clone [email protected]:openjdk/jdk.git
9090
cd jdk
91-
git checkout jdk-17-ga
92-
cd src/utils/hsdis
91+
git checkout jdk-24-ga
9392
# Get a known good binutils
9493
wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz
9594
tar xf binutils-2.35.tar.gz
96-
make BINUTILS=binutils-2.35 ARCH=amd64
97-
sudo cp build/linux-amd64/hsdis-amd64.so /usr/lib/jvm/java-17-openjdk/lib/server/
95+
bash configure --with-hsdis=binutils --with-binutils-src=binutils-2.35 \
96+
--with-boot-jdk=~/.gradle/jdks/oracle_corporation-24-amd64-linux.2
97+
make build-hsdis
98+
cp ./build/linux-x86_64-server-release/jdk/lib/hsdis-amd64.so \
99+
~/.gradle/jdks/oracle_corporation-24-amd64-linux.2/lib/hsdis.so
98100
```
99101

100102
If you want to disassemble a single method do something like this:
@@ -105,6 +107,30 @@ gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDia
105107

106108
If you want `perf` to find the hot methods for you, then do add `-prof perfasm`.
107109

110+
NOTE: `perfasm` will need more access:
111+
```
112+
sudo bash
113+
echo -1 > /proc/sys/kernel/perf_event_paranoid
114+
exit
115+
```
116+
117+
If you get warnings like:
118+
```
119+
The perf event count is suspiciously low (0).
120+
```
121+
then check if you are bumping into [this](https://man.archlinux.org/man/perf-stat.1.en#INTEL_HYBRID_SUPPORT)
122+
by running:
123+
```
124+
perf stat -B dd if=/dev/zero of=/dev/null count=1000000
125+
```
126+
127+
If you see lines like:
128+
```
129+
765019980 cpu_atom/cycles/ # 1.728 GHz (0.60%)
130+
2258845959 cpu_core/cycles/ # 5.103 GHz (99.18%)
131+
```
132+
then `perf` is just not going to work for you.
133+
108134
## Async Profiler
109135

110136
Note: Linux and Mac only. Sorry Windows.

0 commit comments

Comments
 (0)