Skip to content

Commit 2fc52ae

Browse files
original-brownbearoriginal-brownbear
committed
Merge remote-tracking branch 'elastic/main' into compress-delayable-writeable
2 parents 217e0c4 + ec7025c commit 2fc52ae

File tree

31 files changed

+306
-121
lines changed

31 files changed

+306
-121
lines changed

‎.buildkite/pipelines/intake.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ steps:
5656
timeout_in_minutes: 300
5757
matrix:
5858
setup:
59-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
59+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
6060
agents:
6161
provider: gcp
6262
image: family/elasticsearch-ubuntu-2004

‎.buildkite/pipelines/periodic-packaging.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,8 @@ steps:
287287
env:
288288
BWC_VERSION: 8.15.5
289289

290-
- label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
291-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
290+
- label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
291+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
292292
timeout_in_minutes: 300
293293
matrix:
294294
setup:
@@ -301,7 +301,7 @@ steps:
301301
machineType: custom-16-32768
302302
buildDirectory: /dev/shm/bk
303303
env:
304-
BWC_VERSION: 8.16.7
304+
BWC_VERSION: 8.16.6
305305

306306
- label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
307307
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
@@ -319,8 +319,8 @@ steps:
319319
env:
320320
BWC_VERSION: 8.17.6
321321

322-
- label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
323-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
322+
- label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
323+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
324324
timeout_in_minutes: 300
325325
matrix:
326326
setup:
@@ -333,7 +333,7 @@ steps:
333333
machineType: custom-16-32768
334334
buildDirectory: /dev/shm/bk
335335
env:
336-
BWC_VERSION: 8.18.0
336+
BWC_VERSION: 8.18.1
337337

338338
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
339339
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0

‎.buildkite/pipelines/periodic.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -306,8 +306,8 @@ steps:
306306
- signal_reason: agent_stop
307307
limit: 3
308308

309-
- label: 8.16.7 / bwc
310-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
309+
- label: 8.16.6 / bwc
310+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
311311
timeout_in_minutes: 300
312312
agents:
313313
provider: gcp
@@ -316,7 +316,7 @@ steps:
316316
buildDirectory: /dev/shm/bk
317317
preemptible: true
318318
env:
319-
BWC_VERSION: 8.16.7
319+
BWC_VERSION: 8.16.6
320320
retry:
321321
automatic:
322322
- exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
344344
- signal_reason: agent_stop
345345
limit: 3
346346

347-
- label: 8.18.0 / bwc
348-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
347+
- label: 8.18.1 / bwc
348+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
349349
timeout_in_minutes: 300
350350
agents:
351351
provider: gcp
@@ -354,7 +354,7 @@ steps:
354354
buildDirectory: /dev/shm/bk
355355
preemptible: true
356356
env:
357-
BWC_VERSION: 8.18.0
357+
BWC_VERSION: 8.18.1
358358
retry:
359359
automatic:
360360
- exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
486486
setup:
487487
ES_RUNTIME_JAVA:
488488
- openjdk21
489-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
489+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
490490
agents:
491491
provider: gcp
492492
image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
533533
ES_RUNTIME_JAVA:
534534
- openjdk21
535535
- openjdk23
536-
BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
536+
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
537537
agents:
538538
provider: gcp
539539
image: family/elasticsearch-ubuntu-2004

‎.buildkite/scripts/dra-workflow.trigger.sh

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ echo "steps:"
77
source .buildkite/scripts/branches.sh
88

99
for BRANCH in "${BRANCHES[@]}"; do
10-
if [[ "$BRANCH" == "9.0" ]]; then
11-
continue
12-
fi
13-
1410
INTAKE_PIPELINE_SLUG="elasticsearch-intake"
1511
BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
1612
LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')

‎.ci/bwcVersions

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ BWC_VERSION:
1515
- "8.13.4"
1616
- "8.14.3"
1717
- "8.15.5"
18-
- "8.16.7"
18+
- "8.16.6"
1919
- "8.17.6"
20-
- "8.18.0"
20+
- "8.18.1"
2121
- "8.19.0"
2222
- "9.0.1"
2323
- "9.1.0"

‎.ci/snapshotBwcVersions

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
BWC_VERSION:
22
- "8.17.6"
3-
- "8.18.0"
3+
- "8.18.1"
44
- "8.19.0"
55
- "9.0.1"
66
- "9.1.0"

‎build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ rootProject.name = "root"
1111

1212
include ":distribution:bwc:bugfix"
1313
include ":distribution:bwc:bugfix2"
14+
include ":distribution:bwc:bugfix3"
1415
include ":distribution:bwc:minor"
1516
include ":distribution:bwc:major"
1617
include ":distribution:bwc:staged"

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> devel
165165
.sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED))))
166166
.toList();
167167

168-
boolean existingBugfix = false;
168+
int bugfixCount = 0;
169169
boolean existingStaged = false;
170170
for (int i = 0; i < featureFreezeBranches.size(); i++) {
171171
String branch = featureFreezeBranches.get(i);
@@ -198,9 +198,9 @@ private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> devel
198198
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
199199
existingStaged = true;
200200
} else { // This is a bugfix
201-
String project = existingBugfix ? "bugfix2" : "bugfix";
201+
bugfixCount++;
202+
String project = "bugfix" + (bugfixCount > 1 ? bugfixCount : "");
202203
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
203-
existingBugfix = true;
204204
}
205205
}
206206

‎build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ allprojects {
301301
// ensure we have best possible caching of bwc builds
302302
dependsOn ":distribution:bwc:bugfix:buildBwcLinuxTar"
303303
dependsOn ":distribution:bwc:bugfix2:buildBwcLinuxTar"
304+
dependsOn ":distribution:bwc:bugfix3:buildBwcLinuxTar"
304305
dependsOn ":distribution:bwc:minor:buildBwcLinuxTar"
305306
dependsOn ":distribution:bwc:staged:buildBwcLinuxTar"
306307
dependsOn ":distribution:bwc:staged2:buildBwcLinuxTar"

‎distribution/bwc/bugfix3/build.gradle

Whitespace-only changes.

‎docs/changelog/126858.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126858
2+
summary: Leverage threadpool schedule for inference api to avoid long running thread
3+
area: Machine Learning
4+
type: bug
5+
issues:
6+
- 126853

‎docs/changelog/126889.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126889
2+
summary: Rework uniquify to not use iterators
3+
area: Infra/Core
4+
type: bug
5+
issues:
6+
- 126883

‎docs/changelog/126930.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126930
2+
summary: Adding missing `onFailure` call for Inference API start model request
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎docs/changelog/126956.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126956
2+
summary: Temporarily bypass competitive iteration for filters aggregation
3+
area: Aggregations
4+
type: bug
5+
issues: []

‎docs/reference/elasticsearch/rest-apis/retrievers.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -560,11 +560,11 @@ Refer to [*Semantic re-ranking*](docs-content://solutions/search/ranking/semanti
560560

561561
### Prerequisites [_prerequisites_15]
562562

563-
To use `text_similarity_reranker` you must first set up an inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be set up with a machine learning model that can compute text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
563+
To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which is based on [Elastic Rerank](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-rerank.html) and serves as the default if no `inference_id` is provided. This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be configured with a machine learning model capable of computing text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
564564

565565
You have the following options:
566566

567-
* Use the the built-in [Elastic Rerank](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) cross-encoder model via the inference API’s {{es}} service.
567+
* Use the built-in [Elastic Rerank](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) cross-encoder model via the inference API’s {{es}} service. For an example of creating an endpoint using the Elastic Rerank model, refer to [this guide](https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html#inference-example-elastic-reranker).
568568
* Use the [Cohere Rerank inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
569569
* Use the [Google Vertex AI inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
570570
* Upload a model to {{es}} with [Eland](eland://reference/machine-learning.md#ml-nlp-pytorch) using the `text_similarity` NLP task type.
@@ -606,9 +606,9 @@ score = ln(score), if score < 0
606606

607607

608608
`inference_id`
609-
: (Required, `string`)
609+
: (Optional, `string`)
610610

611-
Unique identifier of the inference endpoint created using the {{infer}} API.
611+
Unique identifier of the inference endpoint created using the {{infer}} API. If you don’t specify an inference endpoint, the `inference_id` field defaults to `.rerank-v1-elasticsearch`, a preconfigured endpoint for the elasticsearch `.rerank-v1` model.
612612

613613

614614
`inference_text`

‎server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@
2929
import java.util.Iterator;
3030
import java.util.List;
3131

32+
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
3233
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
3334
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
35+
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
3436
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
3537
import static org.elasticsearch.search.aggregations.AggregationBuilders.filters;
3638
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
@@ -95,6 +97,118 @@ public void setupSuiteScopeCluster() throws Exception {
9597
ensureSearchable();
9698
}
9799

100+
// This test replicates a strange filter query & filters aggregation behavior
101+
// we apparently utilize competitive iterators strangely.
102+
// See: https://github.com/elastic/elasticsearch/issues/126955
103+
public void testSimpleWithFilterQuery() throws Exception {
104+
createIndex("filters_idx");
105+
String groupFieldName = "group";
106+
String subGroupFieldName = "subGroup";
107+
108+
int numTotalGroup0 = 500;
109+
String group0Name = "group0";
110+
111+
int numTotalGroup1 = 1000;
112+
String group1Name = "group1";
113+
114+
int subGroup0 = 100;
115+
String subGroup0Name = "subGroup0";
116+
117+
int subGroup1 = 50;
118+
String subGroup1Name = "subGroup1";
119+
120+
int subGroup2 = 25;
121+
String subGroup2Name = "subGroup2";
122+
int others = 10;
123+
String otherName = "others";
124+
List<IndexRequestBuilder> builders = new ArrayList<>();
125+
for (int i = 0; i < numTotalGroup0; i++) {
126+
for (int j = 0; j < subGroup0; j++) {
127+
XContentBuilder source = jsonBuilder().startObject()
128+
.field(groupFieldName, group0Name)
129+
.field(subGroupFieldName, subGroup0Name)
130+
.endObject();
131+
builders.add(prepareIndex("filters_idx").setSource(source));
132+
}
133+
for (int j = 0; j < subGroup1; j++) {
134+
XContentBuilder source = jsonBuilder().startObject()
135+
.field(groupFieldName, group0Name)
136+
.field(subGroupFieldName, subGroup1Name)
137+
.endObject();
138+
builders.add(prepareIndex("filters_idx").setSource(source));
139+
}
140+
for (int j = 0; j < subGroup2; j++) {
141+
XContentBuilder source = jsonBuilder().startObject()
142+
.field(groupFieldName, group0Name)
143+
.field(subGroupFieldName, subGroup2Name)
144+
.endObject();
145+
builders.add(prepareIndex("filters_idx").setSource(source));
146+
}
147+
for (int j = 0; j < others; j++) {
148+
XContentBuilder source = jsonBuilder().startObject()
149+
.field(groupFieldName, group0Name)
150+
.field(subGroupFieldName, otherName)
151+
.endObject();
152+
builders.add(prepareIndex("filters_idx").setSource(source));
153+
}
154+
}
155+
for (int i = 0; i < numTotalGroup1; i++) {
156+
for (int j = 0; j < subGroup0; j++) {
157+
XContentBuilder source = jsonBuilder().startObject()
158+
.field(groupFieldName, group1Name)
159+
.field(subGroupFieldName, subGroup0Name)
160+
.endObject();
161+
builders.add(prepareIndex("filters_idx").setSource(source));
162+
}
163+
for (int j = 0; j < subGroup1; j++) {
164+
XContentBuilder source = jsonBuilder().startObject()
165+
.field(groupFieldName, group1Name)
166+
.field(subGroupFieldName, subGroup1Name)
167+
.endObject();
168+
builders.add(prepareIndex("filters_idx").setSource(source));
169+
}
170+
for (int j = 0; j < subGroup2; j++) {
171+
XContentBuilder source = jsonBuilder().startObject()
172+
.field(groupFieldName, group1Name)
173+
.field(subGroupFieldName, subGroup2Name)
174+
.endObject();
175+
builders.add(prepareIndex("filters_idx").setSource(source));
176+
}
177+
for (int j = 0; j < others; j++) {
178+
XContentBuilder source = jsonBuilder().startObject()
179+
.field(groupFieldName, group1Name)
180+
.field(subGroupFieldName, otherName)
181+
.endObject();
182+
builders.add(prepareIndex("filters_idx").setSource(source));
183+
}
184+
}
185+
indexRandom(true, false, true, builders);
186+
ensureSearchable();
187+
assertNoFailuresAndResponse(
188+
prepareSearch("filters_idx").setSize(0)
189+
.setRequestCache(false)
190+
.setTrackTotalHits(true)
191+
.setQuery(boolQuery().filter(termQuery(groupFieldName + ".keyword", group0Name)))
192+
.addAggregation(
193+
filters(
194+
"results",
195+
new KeyedFilter(subGroup0Name, termsQuery(subGroupFieldName + ".keyword", subGroup0Name)),
196+
new KeyedFilter(subGroup1Name, termsQuery(subGroupFieldName + ".keyword", subGroup1Name)),
197+
new KeyedFilter(subGroup2Name, termsQuery(subGroupFieldName + ".keyword", subGroup2Name))
198+
// This is key
199+
).otherBucket(false)
200+
),
201+
searchResponse -> {
202+
Filters filters = searchResponse.getAggregations().get("results");
203+
assertThat(filters, notNullValue());
204+
assertThat(filters.getName(), equalTo("results"));
205+
Filters.Bucket bucket = filters.getBucketByKey(subGroup0Name);
206+
assertThat(bucket, Matchers.notNullValue());
207+
assertThat(bucket.getDocCount(), equalTo((long) subGroup0 * numTotalGroup0));
208+
}
209+
);
210+
}
211+
98212
public void testSimple() throws Exception {
99213
assertNoFailuresAndResponse(
100214
prepareSearch("idx").addAggregation(

‎server/src/main/java/org/elasticsearch/TransportVersions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ static TransportVersion def(int id) {
141141
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
142142
public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_18 = def(8_840_0_01);
143143
public static final TransportVersion RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_8_18 = def(8_840_0_02);
144+
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_1 = def(8_840_0_03);
144145
public static final TransportVersion INITIAL_ELASTICSEARCH_8_19 = def(8_841_0_00);
145146
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_841_0_01);
146147
public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_19 = def(8_841_0_02);

‎server/src/main/java/org/elasticsearch/Version.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,6 @@ public class Version implements VersionId<Version>, ToXContentFragment {
195195
public static final Version V_8_16_4 = new Version(8_16_04_99);
196196
public static final Version V_8_16_5 = new Version(8_16_05_99);
197197
public static final Version V_8_16_6 = new Version(8_16_06_99);
198-
public static final Version V_8_16_7 = new Version(8_16_07_99);
199198
public static final Version V_8_17_0 = new Version(8_17_00_99);
200199
public static final Version V_8_17_1 = new Version(8_17_01_99);
201200
public static final Version V_8_17_2 = new Version(8_17_02_99);
@@ -204,6 +203,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
204203
public static final Version V_8_17_5 = new Version(8_17_05_99);
205204
public static final Version V_8_17_6 = new Version(8_17_06_99);
206205
public static final Version V_8_18_0 = new Version(8_18_00_99);
206+
public static final Version V_8_18_1 = new Version(8_18_01_99);
207207
public static final Version V_8_19_0 = new Version(8_19_00_99);
208208
public static final Version V_9_0_0 = new Version(9_00_00_99);
209209
public static final Version V_9_0_1 = new Version(9_00_01_99);

0 commit comments

Comments
 (0)