Merge branch 'main' into vectorized_exhaustive_evaluation

Author: jpountz

.github/actions/prepare-for-build/action.yml

@@ -25,7 +25,13 @@ runs:
         java-version: ${{ inputs.java-version }}
         java-package: jdk
 
-    # This includes "smart" caching of the wrapper and dependencies.
+    - name: Cache gradle-wrapper.jar
+      uses: actions/cache@v4
+      with:
+        path: gradle/wrapper/gradle-wrapper.jar
+        key: gradle-wrapper-${{ hashFiles('gradle/wrapper/gradle-wrapper.jar.sha256') }}
+
+    # This includes "smart" caching of gradle dependencies.
     - name: Set up Gradle
       uses: gradle/actions/setup-gradle@v4
       with:

.github/workflows/verify-changelog-and-set-milestone.yml

@@ -14,6 +14,7 @@ jobs:
   changelog-verifier-and-milestone-setter:
     name: Verify Change Log Entry and Set Milestone
     runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
 
     steps:
       - name: Checkout repository

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -159,6 +159,9 @@ public void run(Path destination) throws IOException, NoSuchAlgorithmException {
         }
 
         switch (connection.getResponseCode()) {
+          case /* TOO_MANY_REQUESTS */ 429:
+          // it may not be possible to recover from this using a short delay
+          // but try anyway.
           case HttpURLConnection.HTTP_INTERNAL_ERROR:
           case HttpURLConnection.HTTP_UNAVAILABLE:
           case HttpURLConnection.HTTP_BAD_GATEWAY:

build.gradle

@@ -200,6 +200,7 @@ apply from: file('gradle/documentation/markdown.gradle')
 apply from: file('gradle/documentation/render-javadoc.gradle')
 apply from: file('gradle/documentation/check-broken-links.gradle')
 
+apply from: file('gradle/hacks/downloader-retries.gradle')
 apply from: file('gradle/hacks/gradle-archives.gradle')
 apply from: file('gradle/hacks/wipe-temp.gradle')
 apply from: file('gradle/hacks/hashmapAssertions.gradle')

gradle/hacks/downloader-retries.gradle

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+allprojects {project ->
+    // Limit configuration to just those projects that actually have the plugin enabled.
+    plugins.matching { Plugin plugin ->
+        return plugin.class.name == "de.undercouch.gradle.tasks.download.DownloadTaskPlugin"
+    }.configureEach {
+        project.tasks.withType(Download).configureEach {
+            it.retries 3
+        }
+    }
+}

gradle/libs.versions.toml

@@ -51,7 +51,7 @@ procfork = "1.0.6"
 # unit tests
 randomizedtesting = "2.8.3"
 # license checks
-rat = "0.14"
+rat = "0.15"
 # spatial-extras/ support
 s2-geometry = "1.0.0"
 # spatial-extras/ support

gradlew

@@ -222,7 +222,12 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
 fi
 
 GRADLE_WRAPPER_JAR="$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
-if ! ( cd "$APP_HOME/gradle/wrapper" && sha256sum --status -c "${GRADLE_WRAPPER_JAR}.sha256" ); then
+if "$darwin"; then
+    shasumcmd=shasum
+else
+    shasumcmd=sha256sum
+fi
+if [ ! -e "$GRADLE_WRAPPER_JAR" ] || ! ( cd "$APP_HOME/gradle/wrapper" && "$shasumcmd" --status -c "${GRADLE_WRAPPER_JAR}.sha256" ); then
     "$JAVACMD" $JAVA_OPTS "$APP_HOME/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "$GRADLE_WRAPPER_JAR"
     WRAPPER_STATUS=$?
     if [ "$WRAPPER_STATUS" -eq 1 ]; then

lucene/CHANGES.txt

@@ -90,6 +90,8 @@ Improvements
 * GITHUB#14602: Refactor the expressions compiler to use official ClassData BSM with indexed lookup
   (Uwe Schindler)
 
+* GITHUB#14443: Support adaptive refresh in Searcher Managers (Vigya Sharma)
+
 Optimizations
 ---------------------
 * GITHUB#14418: Quick exit on filter query matching no docs when rewriting knn query. (Pan Guixin)
@@ -116,7 +118,13 @@ Optimizations
 
 Bug Fixes
 ---------------------
-(No changes)
+* GITHUB#14654: ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when
+  used if the DoubleValuesSource needed scores. (David Smiley)
+
+* GITHUB#14682 : Fix for add char and token filters in Luke Analysis tab. (Amir Raza)
+
+* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentException
+  instead of UnsupportedOperationException when values are out of order. (Shubham Sharma)
 
 Build
 ---------------------

lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java

@@ -194,19 +194,23 @@ private TopDocs getLeafResults(
     final int cost = acceptDocs.cardinality();
     QueryTimeout queryTimeout = timeLimitingKnnCollectorManager.getQueryTimeout();
 
-    if (cost <= k) {
-      // If there are <= k possible matches, short-circuit and perform exact search, since HNSW
-      // must always visit at least k documents
+    float leafProportion = ctx.reader().maxDoc() / (float) ctx.parent.reader().maxDoc();
+    int perLeafTopK = perLeafTopKCalculation(k, leafProportion);
+
+    if (cost <= perLeafTopK) {
+      // If there are <= perLeafTopK possible matches, short-circuit and perform exact search, since
+      // HNSW must always visit at least perLeafTopK documents
       return exactSearch(ctx, new BitSetIterator(acceptDocs, cost), queryTimeout);
     }
 
     // Perform the approximate kNN search
     // We pass cost + 1 here to account for the edge case when we explore exactly cost vectors
     TopDocs results = approximateSearch(ctx, acceptDocs, cost + 1, timeLimitingKnnCollectorManager);
+
     if ((results.totalHits.relation() == TotalHits.Relation.EQUAL_TO
-            // We know that there are more than `k` available docs, if we didn't even get `k`
-            // something weird happened, and we need to drop to exact search
-            && results.scoreDocs.length >= k)
+            // We know that there are more than `perLeafTopK` available docs, if we didn't even get
+            // `perLeafTopK` something weird happened, and we need to drop to exact search
+            && results.scoreDocs.length >= perLeafTopK)
         // Return partial results only when timeout is met
         || (queryTimeout != null && queryTimeout.shouldExit())) {
       return results;

lucene/core/src/java/org/apache/lucene/search/DISIDocIdStream.java

@@ -198,17 +198,7 @@ private int scoreWindow(
     int bitsetWindowMax = (int) Math.min(minDocIDRunEnd, (long) WINDOW_SIZE + min);
 
     if (windowTwoPhases.isEmpty()) {
-      if (acceptDocs == null && windowApproximations.size() == 1) {
-        // We have a range of doc IDs where all matches of an iterator are matches of the
-        // conjunction.
-        DocIdSetIterator iterator = windowApproximations.get(0);
-        if (iterator.docID() < min) {
-          iterator.advance(min);
-        }
-        collector.collect(new DISIDocIdStream(iterator, bitsetWindowMax, clauseWindowMatches));
-      } else {
-        scoreWindowUsingBitSet(collector, acceptDocs, windowApproximations, min, bitsetWindowMax);
-      }
+      scoreWindowUsingBitSet(collector, acceptDocs, windowApproximations, min, bitsetWindowMax);
     } else {
       windowTwoPhases.sort(Comparator.comparingDouble(TwoPhaseIterator::matchCost));
       scoreWindowUsingLeapFrog(

lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java

@@ -94,8 +94,13 @@ public void nextCandidate() {
   @Override
   public KnnSearchStrategy getSearchStrategy() {
     KnnSearchStrategy delegateStrategy = delegate.getSearchStrategy();
-    assert delegateStrategy instanceof KnnSearchStrategy.Hnsw;
-    return new KnnSearchStrategy.Patience(
-        this, ((KnnSearchStrategy.Hnsw) delegateStrategy).filteredSearchThreshold());
+    if (delegateStrategy instanceof KnnSearchStrategy.Hnsw hnswStrategy) {
+      return new KnnSearchStrategy.Patience(this, hnswStrategy.filteredSearchThreshold());
+    } else if (delegateStrategy instanceof KnnSearchStrategy.Seeded seededStrategy) {
+      if (seededStrategy.originalStrategy() instanceof KnnSearchStrategy.Hnsw hnswStrategy) {
+        return new KnnSearchStrategy.Patience(this, hnswStrategy.filteredSearchThreshold());
+      }
+    }
+    return delegateStrategy;
   }
 }

lucene/core/src/java/org/apache/lucene/search/HnswQueueSaturationCollector.java

@@ -43,8 +43,7 @@ public class PatienceKnnVectorQuery extends AbstractKnnVectorQuery {
 
   private final int patience;
   private final double saturationThreshold;
-
-  final AbstractKnnVectorQuery delegate;
+  private AbstractKnnVectorQuery delegate;
 
   /**
    * Construct a new PatienceKnnVectorQuery instance for a float vector field
@@ -234,4 +233,18 @@ public KnnCollector newCollector(
           patience);
     }
   }
+
+  @Override
+  public Query rewrite(IndexSearcher indexSearcher) throws IOException {
+    if (delegate instanceof SeededKnnVectorQuery seededKnnVectorQuery) {
+      // this is required because SeededKnnVectorQuery now requires its own rewriting logic (to
+      // create the seed Weight)
+      delegate =
+          new SeededKnnVectorQuery(
+              seededKnnVectorQuery.delegate,
+              seededKnnVectorQuery.seed,
+              seededKnnVectorQuery.createSeedWeight(indexSearcher));
+    }
+    return super.rewrite(indexSearcher);
+  }
 }

lucene/core/src/java/org/apache/lucene/search/PatienceKnnVectorQuery.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+
+/**
+ * Expert: Interface to supply commit for searcher refresh.
+ *
+ * @lucene.experimental
+ */
+public interface RefreshCommitSupplier {
+
+  /**
+   * Expert: Returns the index commit that searcher should refresh on. A null return value (default)
+   * indicates reader should refresh on the latest commit.
+   *
+   * @param reader DirectoryReader to refresh
+   */
+  default IndexCommit getSearcherRefreshCommit(DirectoryReader reader) throws IOException {
+    return null;
+  }
+}