Skip to content

Commit d927c4b

Browse files
Merge remote-tracking branch 'elastic/main' into batched-exec-short
2 parents 88efd18 + 66432fb commit d927c4b

File tree

45 files changed

+894
-214
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+894
-214
lines changed

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex,
527527
CommonStats stats = new CommonStats();
528528
stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes());
529529
stats.store = new StoreStats();
530-
stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1, 0.123));
530+
stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1, 0.123, 0.234));
531531
return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0);
532532
}
533533

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTe
3535
private static final String BASE_PATH = PREFIX + "base_path";
3636
private static final String CLIENT = "ecs_credentials_client";
3737

38-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
38+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials("*", "s3");
3939

4040
private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
4141
new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials)

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res
3333
private static final String BASE_PATH = PREFIX + "base_path";
3434
private static final String CLIENT = "imdsv1_credentials_client";
3535

36-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
36+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials("*", "s3");
3737

3838
private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
3939
new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials)

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3Res
3333
private static final String BASE_PATH = PREFIX + "base_path";
3434
private static final String CLIENT = "imdsv2_credentials_client";
3535

36-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
36+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials("*", "s3");
3737

3838
private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
3939
new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicCredentials::addValidCredentials)

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public class RepositoryS3StsCredentialsRestIT extends AbstractRepositoryS3RestTe
3232
private static final String BASE_PATH = PREFIX + "base_path";
3333
private static final String CLIENT = "sts_credentials_client";
3434

35-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
35+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials("*", "s3");
3636

3737
private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized);
3838

muted-tests.yml

+6-3
Original file line numberDiff line numberDiff line change
@@ -378,9 +378,6 @@ tests:
378378
- class: org.elasticsearch.smoketest.MlWithSecurityIT
379379
method: test {yaml=ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header}
380380
issue: https://github.com/elastic/elasticsearch/issues/125642
381-
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.GCSRepositoryAnalysisRestIT
382-
method: testRepositoryAnalysis
383-
issue: https://github.com/elastic/elasticsearch/issues/125668
384381
- class: org.elasticsearch.packaging.test.DockerTests
385382
method: test010Install
386383
issue: https://github.com/elastic/elasticsearch/issues/125680
@@ -393,6 +390,12 @@ tests:
393390
- class: org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportActionIT
394391
method: testAlreadyUpToDateDataStream
395392
issue: https://github.com/elastic/elasticsearch/issues/125727
393+
- class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerStressTestIT
394+
method: testMergingFallsBehindAndThenCatchesUp
395+
issue: https://github.com/elastic/elasticsearch/issues/125744
396+
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
397+
method: test {yaml=cat.allocation/10_basic/Node forecasts}
398+
issue: https://github.com/elastic/elasticsearch/issues/125661
396399

397400
# Examples:
398401
#

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2EcsCredentialsIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@
2626

2727
public class DiscoveryEc2EcsCredentialsIT extends DiscoveryEc2ClusterFormationTestCase {
2828

29-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
30-
3129
private static final String PREFIX = getIdentifierPrefix("DiscoveryEc2EcsCredentialsIT");
3230
private static final String REGION = PREFIX + "-region";
3331
private static final String CREDENTIALS_ENDPOINT = "/ecs_credentials_endpoint_" + PREFIX;
3432

33+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(REGION, "ec2");
34+
3535
private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
3636
new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials)
3737
.alternativeCredentialsEndpoints(Set.of(CREDENTIALS_ENDPOINT))

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2InstanceProfileIT.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,27 @@
1515
import fixture.aws.imds.Ec2ImdsServiceBuilder;
1616
import fixture.aws.imds.Ec2ImdsVersion;
1717

18+
import org.elasticsearch.common.util.LazyInitializable;
1819
import org.elasticsearch.discovery.DiscoveryModule;
20+
import org.elasticsearch.test.ESTestCase;
1921
import org.elasticsearch.test.cluster.ElasticsearchCluster;
2022
import org.junit.ClassRule;
2123
import org.junit.rules.RuleChain;
2224
import org.junit.rules.TestRule;
2325

2426
import java.util.List;
27+
import java.util.function.Supplier;
2528

2629
public class DiscoveryEc2InstanceProfileIT extends DiscoveryEc2ClusterFormationTestCase {
2730

28-
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials();
31+
// Lazy-initialized so we can generate it randomly, which is not possible in static context.
32+
private static final Supplier<String> regionSupplier = new LazyInitializable<>(ESTestCase::randomIdentifier)::getOrCompute;
33+
34+
private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(regionSupplier, "ec2");
2935

3036
private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
3137
new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).instanceIdentityDocument(
32-
(builder, params) -> builder.field("region", randomIdentifier())
38+
(builder, params) -> builder.field("region", regionSupplier.get())
3339
).newCredentialsConsumer(dynamicCredentials::addValidCredentials)
3440
);
3541

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.common.Strings;
16+
import org.elasticsearch.common.settings.Settings;
17+
import org.elasticsearch.test.rest.ObjectPath;
18+
19+
import java.io.IOException;
20+
import java.util.Collection;
21+
22+
import static org.elasticsearch.upgrades.SnapshotBasedRecoveryIT.indexDocs;
23+
import static org.hamcrest.Matchers.empty;
24+
import static org.hamcrest.Matchers.equalTo;
25+
import static org.hamcrest.Matchers.not;
26+
27+
public class RunningSnapshotIT extends AbstractRollingUpgradeTestCase {
28+
29+
public RunningSnapshotIT(@Name("upgradedNodes") int upgradedNodes) {
30+
super(upgradedNodes);
31+
}
32+
33+
public void testRunningSnapshotCompleteAfterUpgrade() throws Exception {
34+
final String indexName = "index";
35+
final String repositoryName = "repo";
36+
final String snapshotName = "snapshot";
37+
final var nodeIds = getNodesInfo(client()).keySet();
38+
39+
if (isOldCluster()) {
40+
registerRepository(repositoryName, "fs", randomBoolean(), Settings.builder().put("location", "backup").build());
41+
// create an index to have one shard per node
42+
createIndex(indexName, indexSettings(3, 0).put("index.routing.allocation.total_shards_per_node", 1).build());
43+
ensureGreen(indexName);
44+
if (randomBoolean()) {
45+
indexDocs(indexName, between(10, 50));
46+
}
47+
flush(indexName, true);
48+
// Signal shutdown to prevent snapshot from being completed
49+
putShutdownMetadata(nodeIds);
50+
createSnapshot(repositoryName, snapshotName, false);
51+
assertRunningSnapshot(repositoryName, snapshotName);
52+
} else {
53+
if (isUpgradedCluster()) {
54+
deleteShutdownMetadata(nodeIds);
55+
assertNoShutdownMetadata(nodeIds);
56+
ensureGreen(indexName);
57+
assertBusy(() -> assertCompletedSnapshot(repositoryName, snapshotName));
58+
} else {
59+
assertRunningSnapshot(repositoryName, snapshotName);
60+
}
61+
}
62+
}
63+
64+
private void putShutdownMetadata(Collection<String> nodeIds) throws IOException {
65+
for (String nodeId : nodeIds) {
66+
final Request putShutdownRequest = new Request("PUT", "/_nodes/" + nodeId + "/shutdown");
67+
putShutdownRequest.setJsonEntity("""
68+
{
69+
"type": "remove",
70+
"reason": "test"
71+
}""");
72+
client().performRequest(putShutdownRequest);
73+
}
74+
}
75+
76+
private void deleteShutdownMetadata(Collection<String> nodeIds) throws IOException {
77+
for (String nodeId : nodeIds) {
78+
final Request request = new Request("DELETE", "/_nodes/" + nodeId + "/shutdown");
79+
client().performRequest(request);
80+
}
81+
}
82+
83+
private void assertNoShutdownMetadata(Collection<String> nodeIds) throws IOException {
84+
final ObjectPath responsePath = assertOKAndCreateObjectPath(
85+
client().performRequest(new Request("GET", "/_nodes/" + Strings.collectionToCommaDelimitedString(nodeIds) + "/shutdown"))
86+
);
87+
assertThat(responsePath.evaluate("nodes"), empty());
88+
}
89+
90+
private void assertRunningSnapshot(String repositoryName, String snapshotName) throws IOException {
91+
final Request request = new Request("GET", "/_snapshot/" + repositoryName + "/_current");
92+
final ObjectPath responsePath = assertOKAndCreateObjectPath(client().performRequest(request));
93+
assertThat(responsePath.evaluate("total"), equalTo(1));
94+
assertThat(responsePath.evaluate("snapshots.0.snapshot"), equalTo(snapshotName));
95+
}
96+
97+
private void assertCompletedSnapshot(String repositoryName, String snapshotName) throws IOException {
98+
{
99+
final Request request = new Request("GET", "/_snapshot/" + repositoryName + "/_current");
100+
final ObjectPath responsePath = assertOKAndCreateObjectPath(client().performRequest(request));
101+
assertThat(responsePath.evaluate("total"), equalTo(0));
102+
}
103+
104+
{
105+
final Request request = new Request("GET", "/_snapshot/" + repositoryName + "/" + snapshotName);
106+
final ObjectPath responsePath = assertOKAndCreateObjectPath(client().performRequest(request));
107+
assertThat(responsePath.evaluate("total"), equalTo(1));
108+
assertThat(responsePath.evaluate("snapshots.0.snapshot"), equalTo(snapshotName));
109+
assertThat(responsePath.evaluate("snapshots.0.state"), not(equalTo("IN_PROGRESS")));
110+
}
111+
}
112+
}

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ private static Map<String, Object> search(String index, QueryBuilder query) thro
233233
return responseAsMap;
234234
}
235235

236-
private void indexDocs(String indexName, int numDocs) throws IOException {
236+
static void indexDocs(String indexName, int numDocs) throws IOException {
237237
final StringBuilder bulkBody = new StringBuilder();
238238
for (int i = 0; i < numDocs; i++) {
239239
bulkBody.append("{\"index\":{\"_id\":\"").append(i).append("\"}}\n");

server/src/main/java/org/elasticsearch/TransportVersions.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,9 @@ static TransportVersion def(int id) {
205205
public static final TransportVersion RERANK_COMMON_OPTIONS_ADDED = def(9_037_0_00);
206206
public static final TransportVersion ESQL_REPORT_ORIGINAL_TYPES = def(9_038_00_0);
207207
public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO = def(9_039_0_00);
208-
public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_040_0_00);
208+
public static final TransportVersion PROJECT_ID_IN_SNAPSHOT = def(9_040_0_00);
209+
public static final TransportVersion INDEX_STATS_AND_METADATA_INCLUDE_PEAK_WRITE_LOAD = def(9_041_0_00);
210+
public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_042_0_00);
209211

210212
/*
211213
* STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java

+23
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,29 @@ public static <K, T, M extends Map<K, T>> boolean hasKey(MapDiff<K, T, M> diff,
146146
return false;
147147
}
148148

149+
/**
150+
* Create a new JDK map backed MapDiff by transforming the keys with the provided keyFunction.
151+
* @param diff Original MapDiff to transform
152+
* @param keyFunction Function to transform the key
153+
* @param keySerializer Serializer for the new key
154+
*/
155+
public static <K1, K2, T extends Diffable<T>, M1 extends Map<K1, T>> MapDiff<K2, T, Map<K2, T>> jdkMapDiffWithUpdatedKeys(
156+
MapDiff<K1, T, M1> diff,
157+
Function<K1, K2> keyFunction,
158+
KeySerializer<K2> keySerializer
159+
) {
160+
final List<K2> deletes = diff.getDeletes().stream().map(keyFunction).toList();
161+
final List<Map.Entry<K2, Diff<T>>> diffs = diff.getDiffs()
162+
.stream()
163+
.map(entry -> Map.entry(keyFunction.apply(entry.getKey()), entry.getValue()))
164+
.toList();
165+
final List<Map.Entry<K2, T>> upserts = diff.getUpserts()
166+
.stream()
167+
.map(entry -> Map.entry(keyFunction.apply(entry.getKey()), entry.getValue()))
168+
.toList();
169+
return new MapDiff<>(keySerializer, DiffableValueSerializer.getWriteOnlyInstance(), deletes, diffs, upserts, JdkMapBuilder::new);
170+
}
171+
149172
/**
150173
* Creates a MapDiff that applies a single entry diff to a map
151174
*/

0 commit comments

Comments
 (0)