Merge remote-tracking branch 'elastic/main' into batched-exec-short

Author: original-brownbear

.buildkite/hooks/pre-command

@@ -94,6 +94,14 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
   fi
 fi
 
+# Authenticate to the Docker Hub public read-only registry
+if which docker > /dev/null 2>&1; then
+  DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"
+  DOCKERHUB_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"
+
+  echo "$DOCKERHUB_REGISTRY_PASSWORD" | docker login --username "$DOCKERHUB_REGISTRY_USERNAME" --password-stdin docker.io
+fi
+
 if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
   # Run in the background, while the job continues
   nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &

REST_API_COMPATIBILITY.md

@@ -154,9 +154,11 @@ if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("limit
 
 The above code checks the request's compatible version and if the request has the parameter in question. In this case the deprecation warning is not automatic and requires the developer to manually log the warning. `request.param` is also required since it consumes the value as to avoid the error of unconsumed parameters.
 
-### Testing
+### Testing Backwards Compatibility
 
-The primary means of testing compatibility is via the prior major version's YAML REST tests. The build system will download the latest prior version of the YAML rest tests and execute them against the current cluster version. Prior to execution the tests will be transformed by injecting the correct headers to enable compatibility as well as other custom changes to the tests to allow the tests to pass. These customizations are configured via the build.gradle and happen just prior to test execution. Since the compatibility tests are manipulated version of the tests stored in Github (via the past major version), it is important to find the local (on disk) version for troubleshooting compatibility tests.
+The primary means of testing compatibility is via the prior major version's YAML REST tests. The build system will download the latest prior version of the YAML rest tests and execute them against the current cluster version. For example if you are testing main versioned as 9.0.0 the build system will download the yaml tests in the 8.x branch and execute those against the current cluster version for 9.0.0.
+
+Prior to execution the tests will be transformed by injecting the correct headers to enable compatibility as well as other custom changes to the tests to allow the tests to pass. These customizations are configured via the build.gradle and happen just prior to test execution. Since the compatibility tests are manipulated version of the tests stored in Github (via the past major version), it is important to find the local (on disk) version for troubleshooting compatibility tests.
 
 The tests are wired into the `check` task, so that is the easiest way to test locally prior to committing.  More specifically the task is called `yamlRestCompatTest`. These behave nearly identical to it's non-compat `yamlRestTest` task. The only variance is that the tests are sourced from the prior version branch and the tests go through a transformation phase before execution. The transformation task is `yamlRestCompatTestTransform`.
 
@@ -170,6 +172,36 @@ Since these are a variation of backward compatibility testing, the entire suite
 
 In some cases the prior version of the YAML REST tests are not sufficient to fully test changes. This can happen when the prior version has insufficient test coverage. In those cases, you can simply add more testing to the prior version or you can add custom REST tests that will run along side of the other compatibility tests. These custom tests can be found in the `yamlRestCompatTest` sourceset. Custom REST tests for compatibility will not be modified prior to execution, so the correct headers need to be manually added.
 
+#### Breaking Changes
+
+It is possible to be in a state where you have intentionally made a breaking change and the compatibility tests will fail irrespective of checks for `skip` or `requires` cluster or test features in the current version such as 9.0.0. In this state, assuming the breaking changes are reasonable and agreed upon by the breaking change committee, the correct behavior is to skip the test in the `build.gradle` in 9.0.0. For example, if you make a breaking change that causes the `range/20_synthetic_source/Date range` to break then this test can be disabled temporarily in this file `rest-api-spec/build.gradle` like within this snippet:
+
+```groovy
+tasks.named("yamlRestCompatTestTransform").configure({task ->
+    task.skipTest("range/20_synthetic_source/Date range", "date range breaking change causes tests to produce incorrect values for compatibility")
+    task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
+    task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
+    task.skipTestsByFilePattern("indices.create/synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]")
+})
+```
+
+When skipping a test temporarily in 9.0.0, we have to implement the proper `skip` and `requires` conditions to previous branches, such as 8.latest. After these conditions are implemented in 8.latest, you can re-enable the test in 9.0.0 by removing the `skipTest` condition.
+
+The team implementing the changes can decide how to clean up or modify tests based on how breaking changes were backported. e.g.:
+
+In 8.latest:
+
+* Add `skip` / `requires` conditions to existing tests that check the old behavior. This prevents those tests from failing during backward compatibility or upgrade testing from 8.latest to 9.0.0
+
+In 9.0.0:
+
+* Add `requires` conditions for new tests that validate the updated API or output format
+* Add `skip` conditions for older tests that would break in 9.0.0
+
+#### Test Features
+
+Both cluster and test features exist. Cluster features are meant for new capability and test features can specifically be used to gate and manage `skip` and `requires` yaml test operations.  For more information, see [Versioning.md](docs/internal/Versioning.md#cluster-features). When backporting and using these features they can not overlap in name and must be consistent when backported so that clusters built with these features are compatible.
+
 ### Developer's workflow
 
 There should not be much, if any, deviation in a developers normal workflow to introduce and back-port changes. Changes should be applied in main, then back ported as needed.

docs/changelog/124669.yaml

@@ -0,0 +1,10 @@
+pr: 124669
+summary: Release semantic_text as a GA feature
+area: Mapping
+type: feature
+issues: []
+highlight:
+  title: Release semantic_text as a GA feature
+  body: semantic_text is now an official GA (generally available) feature!
+        This field type allows you to easily set up and perform semantic search with minimal ramp up time.
+  notable: true

docs/changelog/125171.yaml

@@ -0,0 +1,5 @@
+pr: 125171
+summary: Reindex data stream indices on different nodes
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/125244.yaml

@@ -0,0 +1,6 @@
+pr: 125244
+summary: Disable logging in `ClusterFormationFailureHelper` on shutdown
+area: Cluster Coordination
+type: bug
+issues:
+ - 105559

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -6,12 +6,6 @@ mapped_pages:
 
 # Semantic text field type [semantic-text]
 
-
-::::{warning}
-This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features.
-::::
-
-
 The `semantic_text` field type automatically generates embeddings for text content using an inference endpoint. Long passages are [automatically chunked](#auto-text-chunking) to smaller sections to enable the processing of larger corpuses of text.
 
 The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). This field type and the [`semantic` query](/reference/query-languages/query-dsl/query-dsl-semantic-query.md) type make it simpler to perform semantic search on your data. The `semantic_text` field type may also be queried with [match](/reference/query-languages/query-dsl/query-dsl-match-query.md), [sparse_vector](/reference/query-languages/query-dsl/query-dsl-sparse-vector-query.md) or [knn](/reference/query-languages/query-dsl/query-dsl-knn-query.md) queries.

docs/reference/query-languages/query-dsl/query-dsl-semantic-query.md

@@ -6,12 +6,6 @@ mapped_pages:
 
 # Semantic query [query-dsl-semantic-query]
 
-
-::::{warning}
-This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features.
-::::
-
-
 The `semantic` query type enables you to perform [semantic search](docs-content://solutions/search/semantic-search.md) on data stored in a [`semantic_text`](/reference/elasticsearch/mapping-reference/semantic-text.md) field.
 
 

muted-tests.yml

@@ -360,9 +360,6 @@ tests:
 - class: org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT
   method: testErrorRecordingOnRetention
   issue: https://github.com/elastic/elasticsearch/issues/124950
-- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyFromRow SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/124951
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
   method: testStopQueryLocalNoRemotes
   issue: https://github.com/elastic/elasticsearch/issues/124959
@@ -387,9 +384,6 @@ tests:
 - class: org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT
   method: testLifecycleAppliedToFailureStore
   issue: https://github.com/elastic/elasticsearch/issues/124999
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=search/610_function_score/Random}
-  issue: https://github.com/elastic/elasticsearch/issues/125010
 - class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests
   method: testGetDataStreamResponse
   issue: https://github.com/elastic/elasticsearch/issues/125083
@@ -402,6 +396,9 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerTests
   method: testSchedulerCloseWaitsForRunningMerge
   issue: https://github.com/elastic/elasticsearch/issues/125236
+- class: org.elasticsearch.reservedstate.service.ReservedClusterStateServiceTests
+  method: testProcessMultipleChunks
+  issue: https://github.com/elastic/elasticsearch/issues/125305
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/610_function_score.yml

@@ -71,13 +71,55 @@
                 }
               },
               "random_score": {
-                "seed": 10
+                "seed": 10,
+                "field": "uuid"
               }
             }
 
   - length: { hits.hits: 2 }
   - match: { hits.total.value: 2 }
 
+---
+"No field only runs on 9.x":
+  - requires:
+      cluster_features: [ "gte_v9.0.0" ]
+      reason: "empty field works seamlessly (relying on _seqno since 9.x)"
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              text:
+                type: text
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body: { text: "foo bar", uuid: 1234 }
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body: { text: "high bar", uuid: 5678 }
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body: { text: "raise bar", uuid: 9012 }
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body: { text: "raise hands", uuid: 3456 }
+
+  - do:
+      indices.refresh:
+        index: [ test ]
   - do:
       search:
         index: test
@@ -91,10 +133,8 @@
                 }
               },
               "random_score": {
-                "seed": 10,
-                "field": "uuid"
+                "seed": 10
               }
             }
-
   - length: { hits.hits: 2 }
   - match: { hits.total.value: 2 }

server/src/main/java/module-info.java

@@ -7,7 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.internal.CompletionsPostingsFormatExtension;
 import org.elasticsearch.plugins.internal.RestExtension;
 
 /** The Elasticsearch Server Module. */
@@ -291,8 +290,7 @@
         to
             org.elasticsearch.serverless.version,
             org.elasticsearch.serverless.buildinfo,
-            org.elasticsearch.serverless.constants,
-            org.elasticsearch.serverless.codec;
+            org.elasticsearch.serverless.constants;
     exports org.elasticsearch.lucene.analysis.miscellaneous;
     exports org.elasticsearch.lucene.grouping;
     exports org.elasticsearch.lucene.queries;
@@ -401,7 +399,6 @@
             org.elasticsearch.stateless,
             org.elasticsearch.settings.secure,
             org.elasticsearch.serverless.constants,
-            org.elasticsearch.serverless.codec,
             org.elasticsearch.serverless.apifiltering,
             org.elasticsearch.internal.security;
 
@@ -422,7 +419,6 @@
     uses org.elasticsearch.node.internal.TerminationHandlerProvider;
     uses org.elasticsearch.internal.VersionExtension;
     uses org.elasticsearch.internal.BuildExtension;
-    uses CompletionsPostingsFormatExtension;
     uses org.elasticsearch.features.FeatureSpecification;
     uses org.elasticsearch.plugins.internal.LoggingDataProvider;
 

server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java

@@ -68,6 +68,7 @@ public class ClusterFormationFailureHelper {
     private final Runnable logLastFailedJoinAttempt;
     @Nullable // if no warning is scheduled
     private volatile WarningScheduler warningScheduler;
+    private volatile boolean loggingEnabled;
 
     /**
      * Works with the {@link JoinHelper} to log the latest node-join attempt failure and cluster state debug information. Must call
@@ -90,6 +91,11 @@ public ClusterFormationFailureHelper(
         this.clusterCoordinationExecutor = threadPool.executor(Names.CLUSTER_COORDINATION);
         this.clusterFormationWarningTimeout = DISCOVERY_CLUSTER_FORMATION_WARNING_TIMEOUT_SETTING.get(settings);
         this.logLastFailedJoinAttempt = logLastFailedJoinAttempt;
+        this.loggingEnabled = true;
+    }
+
+    public void setLoggingEnabled(boolean enabled) {
+        this.loggingEnabled = enabled;
     }
 
     public boolean isRunning() {
@@ -98,7 +104,7 @@ public boolean isRunning() {
 
     /**
      * Schedules a warning debug message to be logged in 'clusterFormationWarningTimeout' time, and periodically thereafter, until
-     * {@link ClusterFormationState#stop()} has been called.
+     * {@link ClusterFormationFailureHelper#stop()} has been called.
      */
     public void start() {
         assert warningScheduler == null;
@@ -125,7 +131,7 @@ public void onFailure(Exception e) {
 
                 @Override
                 protected void doRun() {
-                    if (isActive()) {
+                    if (isActive() && loggingEnabled) {
                         logLastFailedJoinAttempt.run();
                         logger.warn(
                             "{}; for troubleshooting guidance, see {}",

server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java

@@ -1104,6 +1104,7 @@ protected void doStart() {
             applierState = initialState;
             clusterApplier.setInitialState(initialState);
         }
+        clusterFormationFailureHelper.setLoggingEnabled(true);
     }
 
     public DiscoveryStats stats() {
@@ -1126,6 +1127,7 @@ public void startInitialJoin() {
     protected void doStop() {
         configuredHostsResolver.stop();
         joinValidationService.stop();
+        clusterFormationFailureHelper.setLoggingEnabled(false);
     }
 
     @Override

server/src/main/java/org/elasticsearch/cluster/metadata/ProjectId.java

@@ -54,7 +54,7 @@ public static ProjectId fromId(String id) {
         }
     }
 
-    static boolean isValidFormatId(String id) {
+    public static boolean isValidFormatId(String id) {
         if (id.length() > MAX_LENGTH) {
             return false;
         }

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -25,10 +25,6 @@
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.elasticsearch.internal.CompletionsPostingsFormatExtension;
-import org.elasticsearch.plugins.ExtensionLoader;
-
-import java.util.ServiceLoader;
 
 /**
  * Class that encapsulates the logic of figuring out the most appropriate file format for a given field, across postings, doc values and
@@ -40,6 +36,7 @@ public class PerFieldFormatSupplier {
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
     private static final ES87TSDBDocValuesFormat tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
+    private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 
     private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
     private final MapperService mapperService;
@@ -60,26 +57,13 @@ private PostingsFormat internalGetPostingsFormatForField(String field) {
         if (mapperService != null) {
             Mapper mapper = mapperService.mappingLookup().getMapper(field);
             if (mapper instanceof CompletionFieldMapper) {
-                return CompletionPostingsFormatHolder.POSTINGS_FORMAT;
+                return completionPostingsFormat;
             }
         }
         // return our own posting format using PFOR
         return es812PostingsFormat;
     }
 
-    private static class CompletionPostingsFormatHolder {
-        private static final PostingsFormat POSTINGS_FORMAT = getCompletionPostingsFormat();
-
-        private static PostingsFormat getCompletionPostingsFormat() {
-            String defaultName = "Completion101"; // Caution: changing this name will result in exceptions if a field is created during a
-            // rolling upgrade and the new codec (specified by the name) is not available on all nodes in the cluster.
-            String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(CompletionsPostingsFormatExtension.class))
-                .map(CompletionsPostingsFormatExtension::getFormatName)
-                .orElse(defaultName);
-            return PostingsFormat.forName(codecName);
-        }
-    }
-
     boolean useBloomFilter(String field) {
         if (mapperService == null) {
             return false;