Merge remote-tracking branch 'elastic/main' into painless-savings

Author: original-brownbear

.buildkite/pipelines/periodic.template.yml

@@ -85,7 +85,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -108,7 +107,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             BWC_VERSION: $BWC_LIST
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -416,7 +416,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -439,7 +438,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
         agents:
           provider: gcp

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -25,6 +25,7 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
@@ -39,21 +40,22 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
-import org.gradle.api.model.ObjectFactory;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.Serializable;
 import java.io.Writer;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
-import java.io.Serializable;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 
 /**
  * Checks files for license headers..
@@ -232,7 +234,7 @@ private ClaimStatistic toXmlReportFile(ReportConfiguration config, Writer writer
 
     private static List<String> unapprovedFiles(File xmlReportFile) {
         try {
-            NodeList resourcesNodes = DocumentBuilderFactory.newInstance()
+            NodeList resourcesNodes = createXmlDocumentBuilderFactory()
                 .newDocumentBuilder()
                 .parse(xmlReportFile)
                 .getElementsByTagName("resource");
@@ -249,6 +251,21 @@ private static List<String> unapprovedFiles(File xmlReportFile) {
         }
     }
 
+    private static DocumentBuilderFactory createXmlDocumentBuilderFactory() throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        dbf.setXIncludeAware(false);
+        dbf.setIgnoringComments(true);
+        dbf.setExpandEntityReferences(false);
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        return dbf;
+    }
+
     private static List<Element> elementList(NodeList resourcesNodes) {
         List<Element> nodeList = new ArrayList<>(resourcesNodes.getLength());
         for (int idx = 0; idx < resourcesNodes.getLength(); idx++) {

docs/changelog/106516.yaml

@@ -0,0 +1,5 @@
+pr: 106516
+summary: "ESQL: perform a reduction on the data node"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106836.yaml

@@ -0,0 +1,5 @@
+pr: 106836
+summary: Make int8_hnsw our default index for new dense-vector fields
+area: Mapping
+type: enhancement
+issues: []

docs/reference/mapping/types/dense-vector.asciidoc

@@ -65,7 +65,7 @@ data structure to support fast kNN retrieval through the <<search-api-knn, `knn`
 Unmapped array fields of float elements with size between 128 and 4096 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
 You can override the default similarity by explicitly mapping the field as `dense_vector` with the desired similarity.
 
-Indexing is enabled by default for dense vector fields.
+Indexing is enabled by default for dense vector fields and indexed as `int8_hnsw`.
 When indexing is enabled, you can define the vector similarity to use in kNN search:
 
 [source,console]
@@ -116,7 +116,8 @@ that sacrifices result accuracy for improved speed.
 
 The `dense_vector` type supports quantization to reduce the memory footprint required when <<approximate-knn, searching>> `float` vectors.
 Currently the only quantization method supported is `int8` and provided vectors `element_type` must be `float`. To use
-a quantized index, you can set your index type to `int8_hnsw`.
+a quantized index, you can set your index type to `int8_hnsw`. When indexing `float` vectors, the current default
+index type is `int8_hnsw`.
 
 When using the `int8_hnsw` index, each of the `float` vectors' dimensions are quantized to 1-byte integers. This can
 reduce the memory footprint by as much as 75% at the cost of some accuracy. However, the disk usage can increase by
@@ -240,9 +241,10 @@ expense of slower indexing speed.
 The type of kNN algorithm to use. Can be either any of:
 +
 --
-* `hnsw` - The default storage type. This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] for scalable
+* `hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] for scalable
     approximate kNN search. This supports all `element_type` values.
-* `int8_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar
+* `int8_hnsw` - The default index type for float vectors.
+This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar
 quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint
 by 4x at the cost of some accuracy. See <<dense-vector-quantization, Automatically quantize vectors for kNN search>>.
 * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values.

docs/reference/search/search-your-data/knn-search.asciidoc

@@ -272,6 +272,8 @@ If you want to provide `float` vectors, but want the memory savings of `byte` ve
 internally they are indexed as `byte` vectors. Additionally, the original `float` vectors are still retained
 in the index.
 
+NOTE: The default index type for `dense_vector` is `int8_hnsw`.
+
 To use quantization, you can use the index type `int8_hnsw` object in the `dense_vector` mapping.
 
 [source,console]
@@ -652,7 +654,10 @@ PUT passage_vectors
                 "properties": {
                     "vector": {
                         "type": "dense_vector",
-                        "dims": 2
+                        "dims": 2,
+                        "index_options": {
+                            "type": "hnsw"
+                        }
                     },
                     "text": {
                         "type": "text",

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -8,8 +8,8 @@
 package org.elasticsearch.datastreams;
 
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.admin.indices.diskusage.AnalyzeIndexDiskUsageAction;
 import org.elasticsearch.action.admin.indices.diskusage.AnalyzeIndexDiskUsageRequest;
+import org.elasticsearch.action.admin.indices.diskusage.TransportAnalyzeIndexDiskUsageAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -468,7 +468,7 @@ public void testTrimId() throws Exception {
 
         // Pre check whether _id stored field uses diskspace:
         var diskUsageResponse = client().execute(
-            AnalyzeIndexDiskUsageAction.INSTANCE,
+            TransportAnalyzeIndexDiskUsageAction.TYPE,
             new AnalyzeIndexDiskUsageRequest(new String[] { dataStreamName }, AnalyzeIndexDiskUsageRequest.DEFAULT_INDICES_OPTIONS, true)
         ).actionGet();
         var map = XContentHelper.convertToMap(XContentType.JSON.xContent(), Strings.toString(diskUsageResponse), false);
@@ -510,7 +510,7 @@ public void testTrimId() throws Exception {
 
         // Check the _id stored field uses no disk space:
         diskUsageResponse = client().execute(
-            AnalyzeIndexDiskUsageAction.INSTANCE,
+            TransportAnalyzeIndexDiskUsageAction.TYPE,
             new AnalyzeIndexDiskUsageRequest(new String[] { dataStreamName }, AnalyzeIndexDiskUsageRequest.DEFAULT_INDICES_OPTIONS, true)
         ).actionGet();
         map = XContentHelper.convertToMap(XContentType.JSON.xContent(), Strings.toString(diskUsageResponse), false);

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -19,9 +19,9 @@
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
-import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockAction;
 import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest;
 import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse;
+import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction;
 import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
@@ -738,7 +738,7 @@ private void addIndexBlockOnce(String indexName) {
         transportActionsDeduplicator.executeOnce(
             addIndexBlockRequest,
             new ErrorRecordingActionListener(
-                AddIndexBlockAction.NAME,
+                TransportAddIndexBlockAction.TYPE.name(),
                 indexName,
                 errorStore,
                 Strings.format("Data stream lifecycle service encountered an error trying to mark index [%s] as readonly", indexName),

modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaThreadPoolTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.kibana;
+
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.SystemIndexThreadPoolTests;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+
+public class KibanaThreadPoolTests extends SystemIndexThreadPoolTests {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Set.of(KibanaPlugin.class);
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106957")
+    public void testKibanaThreadPool() {
+        runWithBlockedThreadPools(() -> {
+            // index documents
+            String idToDelete = client().prepareIndex(".kibana").setSource(Map.of("foo", "delete me!")).get().getId();
+            String idToUpdate = client().prepareIndex(".kibana").setSource(Map.of("foo", "update me!")).get().getId();
+
+            // bulk index, delete, and update
+            Client bulkClient = client();
+            BulkResponse response = bulkClient.prepareBulk(".kibana")
+                .add(bulkClient.prepareIndex(".kibana").setSource(Map.of("foo", "search me!")))
+                .add(bulkClient.prepareDelete(".kibana", idToDelete))
+                .add(bulkClient.prepareUpdate().setId(idToUpdate).setDoc(Map.of("foo", "I'm updated!")))
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get();
+            assertNoFailures(response);
+
+            // match-all search
+            assertHitCount(client().prepareSearch(".kibana").setQuery(QueryBuilders.matchAllQuery()), 2);
+        });
+    }
+}

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilders.java

@@ -18,6 +18,10 @@
                 dims: 3
                 index: true
                 similarity: l2_norm
+                index_options:
+                  type: hnsw
+                  m: 16
+                  ef_construction: 100
   - do:
       bulk:
         index: test-float-index

qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml

@@ -40,7 +40,7 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
+import static org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction.TASKS_ORIGIN;
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;

qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java

@@ -23,6 +23,10 @@ setup:
                     dims: 5
                     index: true
                     similarity: l2_norm
+                    index_options:
+                      type: hnsw
+                      m: 16
+                      ef_construction: 200
 
   - do:
       index:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml

@@ -19,6 +19,10 @@ setup:
                 dims: 4
                 index : true
                 similarity : l2_norm
+                index_options:
+                  type: hnsw
+                  m: 16
+                  ef_construction: 200
               my_name:
                 type: keyword
                 store: true

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml

@@ -19,6 +19,10 @@ setup:
                 dims: 4
                 index : true
                 similarity : l2_norm
+                index_options:
+                  type: hnsw
+                  m: 16
+                  ef_construction: 200
               my_name:
                 type: keyword
                 store: true