Skip to content

Commit 513b044

Browse files
original-brownbearoriginal-brownbear
committed
Merge remote-tracking branch 'elastic/main' into check-for-all-leaks-in-rest
2 parents 2561a6e + b2bc951 commit 513b044

File tree

467 files changed

+7146
-3001
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

467 files changed

+7146
-3001
lines changed

‎.buildkite/pipelines/periodic.template.yml

+2
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ steps:
8888
- openjdk17
8989
- openjdk21
9090
- openjdk22
91+
- openjdk23
9192
GRADLE_TASK:
9293
- checkPart1
9394
- checkPart2
@@ -113,6 +114,7 @@ steps:
113114
- openjdk17
114115
- openjdk21
115116
- openjdk22
117+
- openjdk23
116118
BWC_VERSION: $BWC_LIST
117119
agents:
118120
provider: gcp

‎.buildkite/pipelines/periodic.yml

+2
Original file line numberDiff line numberDiff line change
@@ -735,6 +735,7 @@ steps:
735735
- openjdk17
736736
- openjdk21
737737
- openjdk22
738+
- openjdk23
738739
GRADLE_TASK:
739740
- checkPart1
740741
- checkPart2
@@ -760,6 +761,7 @@ steps:
760761
- openjdk17
761762
- openjdk21
762763
- openjdk22
764+
- openjdk23
763765
BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
764766
agents:
765767
provider: gcp

‎distribution/packages/src/deb/lintian/elasticsearch

+4
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,7 @@ unknown-field License
5959
# don't build them ourselves and the license precludes us modifying them
6060
# to fix this.
6161
library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so
62+
63+
# shared-lib-without-dependency-information (now shared-library-lacks-prerequisites) is falsely reported for libvec.so
64+
# which has no dependencies (not even libc) besides the symbols in the base executable.
65+
shared-lib-without-dependency-information usr/share/elasticsearch/lib/platform/linux-x64/libvec.so

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

+12
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
7777
maybeEnableNativeAccess(),
7878
maybeOverrideDockerCgroup(distroType),
7979
maybeSetActiveProcessorCount(nodeSettings),
80+
maybeWorkaroundG1Bug(),
8081
setReplayFile(distroType, isHotspot),
8182
"-Djava.library.path=" + libraryPath,
8283
"-Djna.library.path=" + libraryPath,
@@ -137,6 +138,17 @@ private static String maybeEnableNativeAccess() {
137138
return "";
138139
}
139140

141+
/*
142+
* Only affects 22 and 22.0.1, see https://bugs.openjdk.org/browse/JDK-8329528
143+
*/
144+
private static String maybeWorkaroundG1Bug() {
145+
Runtime.Version v = Runtime.version();
146+
if (v.feature() == 22 && v.update() <= 1) {
147+
return "-XX:+UnlockDiagnosticVMOptions -XX:G1NumCollectionsKeepPinned=10000000";
148+
}
149+
return "";
150+
}
151+
140152
private static String findLibraryPath(Map<String, String> sysprops) {
141153
// working dir is ES installation, so we use relative path here
142154
Path platformDir = Paths.get("lib", "platform");

‎docs/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -1752,6 +1752,7 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
17521752
name: "my_admin_role"
17531753
body: >
17541754
{
1755+
"description": "Grants full access to all management features within the cluster.",
17551756
"cluster": ["all"],
17561757
"indices": [
17571758
{"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}}

‎docs/changelog/106486.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
pr: 106486
2+
summary: Create custom parser for ISO-8601 datetimes
3+
area: Infra/Core
4+
type: enhancement
5+
issues:
6+
- 102063
7+
highlight:
8+
title: New custom parser for ISO-8601 datetimes
9+
body: |-
10+
This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and
11+
`strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the
12+
default Java date-time parsing. Whilst it maintains much of the same behaviour,
13+
the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields
14+
or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser
15+
to parse it. If a large proportion of the input data consists of these invalid strings, this may cause
16+
a small performance degradation. If you wish to force the use of the old parsers regardless,
17+
set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes.

‎docs/changelog/107886.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 107886
2+
summary: Cluster state role mapper file settings service
3+
area: Authorization
4+
type: enhancement
5+
issues: []

‎docs/changelog/108410.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 108410
2+
summary: GeoIP tasks should wait longer for master
3+
area: Ingest Node
4+
type: bug
5+
issues: []

‎docs/changelog/108452.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 108452
2+
summary: Add the rerank task to the Elasticsearch internal inference service
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/108517.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 108517
2+
summary: Forward `indexServiceSafe` exception to listener
3+
area: Transform
4+
type: bug
5+
issues:
6+
- 108418

‎docs/changelog/108518.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 108518
2+
summary: Remove leading is_ prefix from Enterprise geoip docs
3+
area: Ingest Node
4+
type: bug
5+
issues: []

‎docs/changelog/108521.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 108521
2+
summary: Adding override for lintian false positive on `libvec.so`
3+
area: "Packaging"
4+
type: bug
5+
issues:
6+
- 108514

‎docs/changelog/108522.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 108522
2+
summary: Ensure we return non-negative scores when scoring scalar dot-products
3+
area: Vector Search
4+
type: bug
5+
issues: []

‎docs/changelog/108562.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 108562
2+
summary: Add `internalClusterTest` for and fix leak in `ExpandSearchPhase`
3+
area: Search
4+
type: bug
5+
issues:
6+
- 108369

‎docs/changelog/108571.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 108571
2+
summary: Workaround G1 bug for JDK 22 and 22.0.1
3+
area: Infra/CLI
4+
type: bug
5+
issues: []

‎docs/internal/DistributedArchitectureGuide.md

+105-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
1-
# Distributed Area Team Internals
1+
# Distributed Area Internals
22

3-
(Summary, brief discussion of our features)
3+
The Distributed Area contains indexing and coordination systems.
4+
5+
The index path stretches from the user REST command through shard routing down to each individual shard's translog and storage
6+
engine. Reindexing is effectively reading from a source index and writing to a destination index (perhaps on different nodes).
7+
The coordination side includes cluster coordination, shard allocation, cluster autoscaling stats, task management, and cross
8+
cluster replication. Less obvious coordination systems include networking, the discovery plugin system, the snapshot/restore
9+
logic, and shard recovery.
10+
11+
A guide to the general Elasticsearch components can be found [here](https://github.com/elastic/elasticsearch/blob/main/docs/internal/GeneralArchitectureGuide.md).
412

513
# Networking
614

@@ -237,9 +245,101 @@ works in parallel with the storage engine.)
237245

238246
# Autoscaling
239247

240-
(Reactive and proactive autoscaling. Explain that we surface recommendations, how control plane uses it.)
241-
242-
(Sketch / list the different deciders that we have, and then also how we use information from each to make a recommendation.)
248+
The Autoscaling API in ES (Elasticsearch) uses cluster and node level statistics to provide a recommendation
249+
for a cluster size to support the current cluster data and active workloads. ES Autoscaling is paired
250+
with an ES Cloud service that periodically polls the ES elected master node for suggested cluster
251+
changes. The cloud service will add more resources to the cluster based on Elasticsearch's recommendation.
252+
Elasticsearch by itself cannot automatically scale.
253+
254+
Autoscaling recommendations are tailored for the user [based on user defined policies][], composed of data
255+
roles (hot, frozen, etc) and [deciders][]. There's a public [webinar on autoscaling][], as well as the
256+
public [Autoscaling APIs] docs.
257+
258+
Autoscaling's current implementation is based primary on storage requirements, as well as memory capacity
259+
for ML and frozen tier. It does not yet support scaling related to search load. Paired with ES Cloud,
260+
autoscaling only scales upward, not downward, except for ML nodes that do get scaled up _and_ down.
261+
262+
[based on user defined policies]: https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-autoscaling.html
263+
[deciders]: https://www.elastic.co/guide/en/elasticsearch/reference/current/autoscaling-deciders.html
264+
[webinar on autoscaling]: https://www.elastic.co/webinars/autoscaling-from-zero-to-production-seamlessly
265+
[Autoscaling APIs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/autoscaling-apis.html
266+
267+
### Plugin REST and TransportAction entrypoints
268+
269+
Autoscaling is a [plugin][]. All the REST APIs can be found in [autoscaling/rest/][].
270+
`GetAutoscalingCapacityAction` is the capacity calculation operation REST endpoint, as opposed to the
271+
other rest commands that get/set/delete the policies guiding the capacity calculation. The Transport
272+
Actions can be found in [autoscaling/action/], where [TransportGetAutoscalingCapacityAction][] is the
273+
entrypoint on the master node for calculating the optimal cluster resources based on the autoscaling
274+
policies.
275+
276+
[plugin]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java#L72
277+
[autoscaling/rest/]: https://github.com/elastic/elasticsearch/tree/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/rest
278+
[autoscaling/action/]: https://github.com/elastic/elasticsearch/tree/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action
279+
[TransportGetAutoscalingCapacityAction]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java#L82-L98
280+
281+
### How cluster capacity is determined
282+
283+
[AutoscalingMetadata][] implements [Metadata.Custom][] in order to persist autoscaling policies. Each
284+
Decider is an implementation of [AutoscalingDeciderService][]. The [AutoscalingCalculateCapacityService][]
285+
is responsible for running the calculation.
286+
287+
[TransportGetAutoscalingCapacityAction.computeCapacity] is the entry point to [AutoscalingCalculateCapacityService.calculate],
288+
which creates a [AutoscalingDeciderResults][] for [each autoscaling policy][]. [AutoscalingDeciderResults.toXContent][] then
289+
determines the [maximum required capacity][] to return to the caller. [AutoscalingCapacity][] is the base unit of a cluster
290+
resources recommendation.
291+
292+
The `TransportGetAutoscalingCapacityAction` response is cached to prevent concurrent callers
293+
overloading the system: the operation is expensive. `TransportGetAutoscalingCapacityAction` contains
294+
a [CapacityResponseCache][]. `TransportGetAutoscalingCapacityAction.masterOperation`
295+
calls [through the CapacityResponseCache][], into the `AutoscalingCalculateCapacityService`, to handle
296+
concurrent callers.
297+
298+
[AutoscalingMetadata]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java#L38
299+
[Metadata.Custom]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L141-L145
300+
[AutoscalingDeciderService]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java#L16-L19
301+
[AutoscalingCalculateCapacityService]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java#L43
302+
303+
[TransportGetAutoscalingCapacityAction.computeCapacity]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java#L102-L108
304+
[AutoscalingCalculateCapacityService.calculate]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java#L108-L139
305+
[AutoscalingDeciderResults]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResults.java#L34-L38
306+
[each autoscaling policy]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java#L124-L131
307+
[AutoscalingDeciderResults.toXContent]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResults.java#L78
308+
[maximum required capacity]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResults.java#L105-L116
309+
[AutoscalingCapacity]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java#L27-L35
310+
311+
[CapacityResponseCache]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java#L44-L47
312+
[through the CapacityResponseCache]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java#L97
313+
314+
### Where the data comes from
315+
316+
The Deciders each pull data from different sources as needed to inform their decisions. The
317+
[DiskThresholdMonitor][] is one such data source. The Monitor runs on the master node and maintains
318+
lists of nodes that exceed various disk size thresholds. [DiskThresholdSettings][] contains the
319+
threshold settings with which the `DiskThresholdMonitor` runs.
320+
321+
[DiskThresholdMonitor]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java#L53-L58
322+
[DiskThresholdSettings]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java#L24-L27
323+
324+
### Deciders
325+
326+
The `ReactiveStorageDeciderService` tracks information that demonstrates storage limitations are causing
327+
problems in the cluster. It uses [an algorithm defined here][]. Some examples are
328+
- information from the `DiskThresholdMonitor` to find out whether nodes are exceeding their storage capacity
329+
- number of unassigned shards that failed allocation because of insufficient storage
330+
- the max shard size and minimum node size, and whether these can be satisfied with the existing infrastructure
331+
332+
[an algorithm defined here]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java#L158-L176
333+
334+
The `ProactiveStorageDeciderService` maintains a forecast window that [defaults to 30 minutes][]. It only
335+
runs on data streams (ILM, rollover, etc), not regular indexes. It looks at past [index changes][] that
336+
took place within the forecast window to [predict][] resources that will be needed shortly.
337+
338+
[defaults to 30 minutes]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java#L32
339+
[index changes]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java#L79-L83
340+
[predict]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java#L85-L95
341+
342+
There are several more Decider Services, implementing the `AutoscalingDeciderService` interface.
243343

244344
# Snapshot / Restore
245345

‎docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ Returns information about all stored connector sync jobs ordered by their creati
3030
(Optional, integer) The offset from the first result to fetch. Defaults to `0`.
3131

3232
`status`::
33-
(Optional, job status) The job status the fetched sync jobs need to have.
33+
(Optional, job status) A comma-separated list of job statuses to filter the results. Available statuses include: `canceling`, `canceled`, `completed`, `error`, `in_progress`, `pending`, `suspended`.
3434

3535
`connector_id`::
3636
(Optional, string) The connector id the fetched sync jobs need to have.
3737

3838
`job_type`::
39-
(Optional, job type) A comma-separated list of job types.
39+
(Optional, job type) A comma-separated list of job types. Available job types are: `full`, `incremental` and `access_control`.
4040

4141
[[list-connector-sync-jobs-api-example]]
4242
==== {api-examples-title}

‎docs/reference/data-streams/change-mappings-and-settings.asciidoc

+1-1
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,7 @@ stream's oldest backing index.
602602
// TESTRESPONSE[s/"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
603603
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/]
604604
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
605-
// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
605+
// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/]
606606

607607
<1> First item in the `indices` array for `my-data-stream`. This item contains
608608
information about the stream's oldest backing index,

‎docs/reference/data-streams/downsampling-manual.asciidoc

+1-1
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ This returns:
389389
// TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/]
390390
// TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/]
391391
// TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/]
392-
// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_indices":[],"failure_store":false/]
392+
// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/]
393393
<1> The backing index for this data stream.
394394

395395
Before a backing index can be downsampled, the TSDS needs to be rolled over and

0 commit comments

Comments
 (0)