Merge remote-tracking branch 'elastic/main' into top-hits-via-cleaner

Author: original-brownbear

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -8,16 +8,19 @@
 
 package org.elasticsearch.benchmark.vector;
 
+import org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.ScalarQuantizedRandomVectorScorer;
 import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.vec.VectorScorer;
 import org.elasticsearch.vec.VectorScorerFactory;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -71,10 +74,10 @@ public class VectorScorerBenchmark {
     float vec2Offset;
     float scoreCorrectionConstant;
 
-    ScalarQuantizedVectorSimilarity luceneDotScorer;
-    ScalarQuantizedVectorSimilarity luceneSqrScorer;
-    VectorScorer nativeDotScorer;
-    VectorScorer nativeSqrScorer;
+    RandomVectorScorer luceneDotScorer;
+    RandomVectorScorer luceneSqrScorer;
+    RandomVectorScorer nativeDotScorer;
+    RandomVectorScorer nativeSqrScorer;
 
     @Setup
     public void setup() throws IOException {
@@ -107,14 +110,22 @@ public void setup() throws IOException {
             out.writeInt(Float.floatToIntBits(vec2Offset));
         }
         in = dir.openInput("vector.data", IOContext.DEFAULT);
+        var values = vectorValues(dims, 2, in);
 
-        luceneDotScorer = ScalarQuantizedVectorSimilarity.fromVectorSimilarity(
-            VectorSimilarityFunction.DOT_PRODUCT,
-            scoreCorrectionConstant
+        luceneDotScorer = new ScalarQuantizedRandomVectorScorer(
+            ScalarQuantizedVectorSimilarity.fromVectorSimilarity(VectorSimilarityFunction.DOT_PRODUCT, scoreCorrectionConstant),
+            values.copy(),
+            vec1,
+            vec1Offset
         );
-        luceneSqrScorer = ScalarQuantizedVectorSimilarity.fromVectorSimilarity(VectorSimilarityFunction.EUCLIDEAN, scoreCorrectionConstant);
-        nativeDotScorer = factory.getInt7ScalarQuantizedVectorScorer(dims, size, scoreCorrectionConstant, DOT_PRODUCT, in).get();
-        nativeSqrScorer = factory.getInt7ScalarQuantizedVectorScorer(dims, size, scoreCorrectionConstant, EUCLIDEAN, in).get();
+        luceneSqrScorer = new ScalarQuantizedRandomVectorScorer(
+            ScalarQuantizedVectorSimilarity.fromVectorSimilarity(VectorSimilarityFunction.EUCLIDEAN, scoreCorrectionConstant),
+            values.copy(),
+            vec1,
+            vec1Offset
+        );
+        nativeDotScorer = factory.getInt7ScalarQuantizedVectorScorer(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer(0);
+        nativeSqrScorer = factory.getInt7ScalarQuantizedVectorScorer(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer(0);
 
         // sanity
         var f1 = dotProductLucene();
@@ -144,13 +155,13 @@ public void teardown() throws IOException {
     }
 
     @Benchmark
-    public float dotProductLucene() {
-        return luceneDotScorer.score(vec1, vec1Offset, vec2, vec2Offset);
+    public float dotProductLucene() throws IOException {
+        return luceneDotScorer.score(1);
     }
 
     @Benchmark
     public float dotProductNative() throws IOException {
-        return nativeDotScorer.score(0, 1);
+        return nativeDotScorer.score(1);
     }
 
     @Benchmark
@@ -166,13 +177,13 @@ public float dotProductScalar() {
     // -- square distance
 
     @Benchmark
-    public float squareDistanceLucene() {
-        return luceneSqrScorer.score(vec1, vec1Offset, vec2, vec2Offset);
+    public float squareDistanceLucene() throws IOException {
+        return luceneSqrScorer.score(1);
     }
 
     @Benchmark
     public float squareDistanceNative() throws IOException {
-        return nativeSqrScorer.score(0, 1);
+        return nativeSqrScorer.score(1);
     }
 
     @Benchmark
@@ -186,6 +197,10 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
+    RandomAccessQuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in) throws IOException {
+        return new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(dims, size, in.slice("values", 0, in.length()));
+    }
+
     // Unsigned int7 byte vectors have values in the range of 0 to 127 (inclusive).
     static final byte MIN_INT7_VALUE = 0;
     static final byte MAX_INT7_VALUE = 127;

docs/changelog/108746.yaml

@@ -0,0 +1,5 @@
+pr: 108746
+summary: Support synthetic source for `aggregate_metric_double` when ignore_malf…
+area: Mapping
+type: feature
+issues: []

docs/changelog/108878.yaml

@@ -0,0 +1,5 @@
+pr: 108878
+summary: Support arrays in fallback synthetic source implementation
+area: Mapping
+type: feature
+issues: []

docs/changelog/108881.yaml

@@ -0,0 +1,5 @@
+pr: 108881
+summary: Add synthetic source support for `geo_shape` via fallback implementation
+area: Mapping
+type: feature
+issues: []

docs/changelog/108931.yaml

@@ -0,0 +1,5 @@
+pr: 108931
+summary: Guard systemd library lookup from unreadable directories
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/108942.yaml

@@ -0,0 +1,5 @@
+pr: 108942
+summary: Fix NPE in trained model assignment updater
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/108999.yaml

@@ -0,0 +1,5 @@
+pr: 108999
+summary: Use default translog durability on AD results index
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/109020.yaml

@@ -0,0 +1,6 @@
+pr: 109020
+summary: Only skip deleting a downsampled index if downsampling is in progress as
+  part of DSL retention
+area: Data streams
+type: bug
+issues: []

docs/reference/esql/esql-async-query-api.asciidoc

@@ -24,8 +24,7 @@ POST /_query/async
     | SORT year
     | LIMIT 5
   """,
-  "wait_for_completion_timeout": "2s",
-  "version": "2024.04.01"
+  "wait_for_completion_timeout": "2s"
 }
 ----
 // TEST[setup:library]

docs/reference/esql/esql-query-api.asciidoc

@@ -16,8 +16,7 @@ POST /_query
     | STATS MAX(page_count) BY year
     | SORT year
     | LIMIT 5
-  """,
-  "version": "2024.04.01"
+  """
 }
 ----
 // TEST[setup:library]
@@ -79,10 +78,6 @@ For syntax, refer to <<esql-locale-param>>.
 `query`::
 (Required, string) {esql} query to run. For syntax, refer to <<esql-syntax>>.
 
-`version`::
-(Required, string) {esql} language version. Can be sent in short or long form, e.g.
-`2024.04.01` or `2024.04.01.🚀`. See <<esql-version>> for details.
-
 [discrete]
 [role="child_attributes"]
 [[esql-query-api-response-body]]

docs/reference/esql/esql-rest.asciidoc

@@ -16,8 +16,7 @@ The <<esql-query-api,{esql} query API>> accepts an {esql} query string in the
 ----
 POST /_query?format=txt
 {
-  "query": "FROM library | KEEP author, name, page_count, release_date | SORT page_count DESC | LIMIT 5",
-  "version": "2024.04.01"
+  "query": "FROM library | KEEP author, name, page_count, release_date | SORT page_count DESC | LIMIT 5"
 }
 ----
 // TEST[setup:library]
@@ -56,8 +55,7 @@ POST /_query?format=txt
     | KEEP author, name, page_count, release_date
     | SORT page_count DESC
     | LIMIT 5
-  """,
-  "version": "2024.04.01"
+  """
 }
 ----
 // TEST[setup:library]
@@ -145,8 +143,7 @@ POST /_query?format=txt
         "lte": 200
       }
     }
-  },
-  "version": "2024.04.01"
+  }
 }
 ----
 // TEST[setup:library]
@@ -182,8 +179,7 @@ POST /_query?format=json
     | SORT page_count DESC
     | LIMIT 5
   """,
-  "columnar": true,
-  "version": "2024.04.01"
+  "columnar": true
 }
 ----
 // TEST[setup:library]
@@ -230,8 +226,7 @@ POST /_query
           | EVAL birth_date = date_parse(birth_date_string)
           | EVAL month_of_birth = DATE_FORMAT("MMMM",birth_date)
           | LIMIT 5
-   """,
-  "version": "2024.04.01"
+   """
 }
 ----
 // TEST[setup:library]
@@ -254,8 +249,7 @@ POST /_query
     | STATS count = COUNT(*) by year
     | WHERE count > 0
     | LIMIT 5
-  """,
-  "version": "2024.04.01"
+  """
 }
 ----
 // TEST[setup:library]
@@ -276,8 +270,7 @@ POST /_query
     | WHERE count > ?
     | LIMIT 5
   """,
-  "params": [300, "Frank Herbert", 0],
-  "version": "2024.04.01"
+  "params": [300, "Frank Herbert", 0]
 }
 ----
 // TEST[setup:library]
@@ -311,8 +304,7 @@ POST /_query/async
     | SORT year
     | LIMIT 5
   """,
-  "wait_for_completion_timeout": "2s",
-  "version": "2024.04.01"
+  "wait_for_completion_timeout": "2s"
 }
 ----
 // TEST[setup:library]

docs/reference/esql/esql-using.asciidoc

@@ -18,12 +18,8 @@ Using {esql} to query across multiple clusters.
 <<esql-task-management>>::
 Using the <<tasks,task management API>> to list and cancel {esql} queries.
 
-<<esql-version>>::
-Information about {esql} language versions.
-
 include::esql-rest.asciidoc[]
 include::esql-kibana.asciidoc[]
 include::esql-security-solution.asciidoc[]
 include::esql-across-clusters.asciidoc[]
 include::task-management.asciidoc[]
-include::esql-version.asciidoc[]

docs/reference/esql/esql-version.asciidoc

@@ -17,8 +17,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | LIMIT 2",
-  "version": "2024.04.01"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -66,8 +65,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | LIMIT 2",
-  "version": "2024.04.01"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -108,8 +106,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | LIMIT 2",
-  "version": "2024.04.01"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -151,8 +148,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | EVAL b=TO_STRING(b) | LIMIT 2",
-  "version": "2024.04.01"
+  "query": "FROM mv | EVAL b=TO_STRING(b) | LIMIT 2"
 }
 ----
 
@@ -175,7 +171,7 @@ POST /_query
 ==== Functions
 
 Unless otherwise documented functions will return `null` when applied to a multivalued
-field. This behavior may change in a later version.
+field.
 
 [source,console,id=esql-multivalued-fields-mv-into-null]
 ----
@@ -190,8 +186,7 @@ POST /mv/_bulk?refresh
 ----
 POST /_query
 {
-  "query": "FROM mv | EVAL b + 2, a + b | LIMIT 4",
-  "version": "2024.04.01"
+  "query": "FROM mv | EVAL b + 2, a + b | LIMIT 4"
 }
 ----
 // TEST[continued]
@@ -230,8 +225,7 @@ Work around this limitation by converting the field to single value with one of:
 ----
 POST /_query
 {
-  "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b | LIMIT 4",
-  "version": "2024.04.01"
+  "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b | LIMIT 4"
 }
 ----
 // TEST[continued]