Merge remote-tracking branch 'elastic/main' into batched-exec-short

Author: original-brownbear

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -36,7 +36,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");

docs/changelog/124062.yaml

@@ -0,0 +1,5 @@
+pr: 125404
+summary: Check if the anomaly results index has been rolled over
+area: Machine Learning
+type: upgrade
+issues: []

docs/changelog/125404.yaml

@@ -0,0 +1,6 @@
+pr: 125659
+summary: Non existing synonyms sets do not fail shard recovery for indices
+area: "Analysis"
+type: bug
+issues:
+ - 125603

docs/changelog/125659.yaml

@@ -0,0 +1,5 @@
+pr: 125690
+summary: Take double parameter markers for identifiers out of snapshot
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/125690.yaml

@@ -42,7 +42,7 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind
 
 ## Example
 
-`LOOKUP JOIN` has left-join behavior. If no rows match in the looked index, `LOOKUP JOIN` retains the incoming row and adds `null`s. If many rows in the lookedup index match, `LOOKUP JOIN` adds one row per match.
+`LOOKUP JOIN` has left-join behavior. If no rows match in the lookup index, `LOOKUP JOIN` retains the incoming row and adds `null`s. If many rows in the lookup index match, `LOOKUP JOIN` adds one row per match.
 
 In this example, we have two sample tables:
 
@@ -112,7 +112,9 @@ To use `LOOKUP JOIN`, the following requirements must be met:
   * Numeric types follow these compatibility rules:
     * `short` and `byte` are compatible with `integer` (all represented as `int`)
     * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`)
-  * For text fields: You can use text fields on the left-hand side of the join only if they have a `.keyword` subfield
+  * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield
+
+To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/esql-functions-operators.md#esql-type-conversion-functions) if needed.
 
 For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types).
 
@@ -121,7 +123,7 @@ For a complete list of supported data types and their internal representations,
 The following are the current limitations with `LOOKUP JOIN`
 
 * Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded.
-* Cross cluster search is unsupported. Both source and lookup indices must be local.
+* Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
 * `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.
 * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve.

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -72,7 +72,7 @@ public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexC
                     );
                 } else {
                     reader = new ReaderWithOrigin(
-                        Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService),
+                        Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService, factory.lenient),
                         "[" + synonymsSet + "] synonyms_set in .synonyms index",
                         synonymsSet
                     );

gradle/verification-metadata.xml

@@ -18,44 +18,40 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
-  classname = 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
+  classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
 }
 
 dependencies {
-  // dependencies consistent with 'com.google.cloud:google-cloud-storage-bom:2.50.0'
-  implementation 'com.google.cloud:google-cloud-storage:2.50.0'
-  implementation 'com.google.cloud:google-cloud-core:2.53.1'
-  implementation 'com.google.cloud:google-cloud-core-http:2.53.1'
-  runtimeOnly 'com.google.guava:guava:33.4.0-jre'
-  runtimeOnly 'com.google.guava:failureaccess:1.0.2'
-  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" // 2.0.16 in bom
-  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" // 1.18.0 in bom
-  implementation 'com.google.api:api-common:2.46.1'
-  implementation 'com.google.api:gax:2.63.1'
-  implementation 'org.threeten:threetenbp:1.7.0'
-  runtimeOnly "com.google.protobuf:protobuf-java-util:${versions.protobuf}" // 3.25.5 in bom
-  runtimeOnly "com.google.protobuf:protobuf-java:${versions.protobuf}"
-  runtimeOnly 'com.google.code.gson:gson:2.12.1'
-  runtimeOnly 'com.google.api.grpc:proto-google-common-protos:2.54.1'
-  runtimeOnly 'com.google.api.grpc:proto-google-iam-v1:1.49.1'
-  implementation 'com.google.auth:google-auth-library-credentials:1.33.1'
-  implementation 'com.google.auth:google-auth-library-oauth2-http:1.33.1'
-  runtimeOnly "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}" // 1.37.0 in bom
-  implementation 'com.google.api-client:google-api-client:2.7.2'
-  implementation 'com.google.http-client:google-http-client:1.46.3'
-  runtimeOnly 'com.google.http-client:google-http-client-gson:1.46.3'
-  runtimeOnly 'com.google.http-client:google-http-client-appengine:1.46.3'
-  runtimeOnly 'com.google.http-client:google-http-client-jackson2:1.46.3'
-  runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // 2.18.2 in bom
-  runtimeOnly 'com.google.api:gax-httpjson:2.63.1'
-  runtimeOnly 'io.opencensus:opencensus-api:0.31.1'
-  runtimeOnly 'io.opencensus:opencensus-contrib-http-util:0.31.1'
-  implementation 'com.google.apis:google-api-services-storage:v1-rev20250224-2.0.0'
-  implementation 'org.checkerframework:checker-qual:3.49.0'
-  runtimeOnly 'io.opentelemetry:opentelemetry-api:1.47.0'
-  runtimeOnly 'io.opentelemetry:opentelemetry-context:1.47.0'
-  runtimeOnly 'com.google.api.grpc:proto-google-cloud-storage-v2:2.50.0'
-  runtimeOnly 'io.grpc:grpc-api:1.70.0'
+  api 'com.google.cloud:google-cloud-storage:2.13.1'
+  api 'com.google.cloud:google-cloud-core:2.8.28'
+  api 'com.google.cloud:google-cloud-core-http:2.8.28'
+  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
+  runtimeOnly 'com.google.guava:failureaccess:1.0.1'
+  api "commons-logging:commons-logging:${versions.commonslogging}"
+  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  api "commons-codec:commons-codec:${versions.commonscodec}"
+  api 'com.google.api:api-common:2.3.1'
+  api 'com.google.api:gax:2.20.1'
+  api 'org.threeten:threetenbp:1.6.5'
+  api "com.google.protobuf:protobuf-java-util:${versions.protobuf}"
+  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
+  api 'com.google.code.gson:gson:2.10'
+  api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
+  api 'com.google.auth:google-auth-library-credentials:1.11.0'
+  api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
+  api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}"
+  api 'com.google.api-client:google-api-client:2.1.1'
+  api 'com.google.http-client:google-http-client:1.42.3'
+  api 'com.google.http-client:google-http-client-gson:1.42.3'
+  api 'com.google.http-client:google-http-client-appengine:1.42.3'
+  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
+  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  api 'com.google.api:gax-httpjson:0.105.1'
+  api 'io.grpc:grpc-context:1.49.2'
+  api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+  api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
 
   testImplementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -66,7 +62,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'cluster', 'nodes', 'snapshot', 'indices', 'index', 'bulk', 'count'
+    include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
   }
 }
 
@@ -127,6 +123,11 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
+    // commons-logging optional dependencies
+    'org.apache.avalon.framework.logger.Logger',
+    'org.apache.log.Hierarchy',
+    'org.apache.log.Logger',
+    'javax.jms.Message',
 
     // optional apache http client dependencies
     'org.apache.http.ConnectionReuseStrategy',
@@ -170,59 +171,19 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.http.protocol.HttpProcessor',
     'org.apache.http.protocol.HttpRequestExecutor',
 
-    // grpc/proto stuff
-    'com.google.api.gax.grpc.GrpcCallContext',
-    'com.google.api.gax.grpc.GrpcCallSettings',
-    'com.google.api.gax.grpc.GrpcCallSettings$Builder',
-    'com.google.api.gax.grpc.GrpcInterceptorProvider',
-    'com.google.api.gax.grpc.GrpcStatusCode',
-    'com.google.api.gax.grpc.GrpcStubCallableFactory',
-    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider',
-    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder',
-    'com.google.cloud.grpc.GrpcTransportOptions',
-    'com.google.cloud.grpc.GrpcTransportOptions$Builder',
-    'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter',
-    'com.google.cloud.opentelemetry.metric.MetricConfiguration',
-    'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder',
-    'com.google.storage.v2.StorageClient',
-    'com.google.storage.v2.StorageClient$ListBucketsPagedResponse',
-    'com.google.storage.v2.StorageSettings',
-    'com.google.storage.v2.StorageSettings$Builder',
-    'com.google.storage.v2.stub.GrpcStorageStub',
-    'com.google.storage.v2.stub.StorageStubSettings',
-    // opentelemetry implementation stuff
-    'io.grpc.opentelemetry.GrpcOpenTelemetry',
-    'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder',
-    'io.grpc.protobuf.ProtoUtils',
-    'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider',
-    'io.opentelemetry.sdk.OpenTelemetrySdk',
-    'io.opentelemetry.sdk.OpenTelemetrySdkBuilder',
-    'io.opentelemetry.sdk.common.CompletableResultCode',
-    'io.opentelemetry.sdk.common.export.MemoryMode',
-    'io.opentelemetry.sdk.metrics.Aggregation',
-    'io.opentelemetry.sdk.metrics.InstrumentSelector',
-    'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder',
-    'io.opentelemetry.sdk.metrics.InstrumentType',
-    'io.opentelemetry.sdk.metrics.SdkMeterProvider',
-    'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder',
-    'io.opentelemetry.sdk.metrics.View',
-    'io.opentelemetry.sdk.metrics.ViewBuilder',
-    'io.opentelemetry.sdk.metrics.data.AggregationTemporality',
-    'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector',
-    'io.opentelemetry.sdk.metrics.export.MetricExporter',
-    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader',
-    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder',
-    'io.opentelemetry.sdk.resources.Resource',
+    // commons-logging provided dependencies
+    'javax.servlet.ServletContextEvent',
+    'javax.servlet.ServletContextListener'
   )
 
 
-  if (buildParams.graalVmRuntime == false) {
+  if(buildParams.graalVmRuntime == false) {
     ignoreMissingClasses(
-      'org.graalvm.nativeimage.hosted.Feature',
-      'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
-      'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
-      'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
-      'org.graalvm.nativeimage.hosted.RuntimeReflection'
+        'org.graalvm.nativeimage.hosted.Feature',
+        'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+        'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+        'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+        'org.graalvm.nativeimage.hosted.RuntimeReflection'
     )
   }
 }
@@ -252,7 +213,7 @@ Map<String, Object> expansions = [
 
 tasks.named("processYamlRestTestResources").configure {
   inputs.properties(expansions)
-  filter("tokens": expansions, ReplaceTokens.class)
+  filter("tokens" : expansions, ReplaceTokens.class)
 }
 
 tasks.named("internalClusterTest").configure {

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java

@@ -0,0 +1,5 @@
+Apache Commons CLI
+Copyright 2001-2009 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 1999-2005 The Apache Software Foundation
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,4 +200,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-

modules/repository-gcs/build.gradle

@@ -0,0 +1,20 @@
+Apache Log4j
+Copyright 1999-2023 Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+ResolverUtil.java
+Copyright 2005-2006 Tim Fennell
+
+Dumbster SMTP test server
+Copyright 2004 Jason Paul Kitchen
+
+TypeUtil.java
+Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams
+
+picocli (http://picocli.info)
+Copyright 2017 Remko Popma
+
+TimeoutBlockingWaitStrategy.java and parts of Util.java
+Copyright 2011 LMAX Ltd.

modules/repository-gcs/licenses/checker-qual-LICENSE.txt

@@ -75,9 +75,9 @@
 class GoogleCloudStorageBlobStore implements BlobStore {
 
     /**
-     * see {@link com.google.cloud.storage.BaseStorageWriteChannel#chunkSize}
+     * see com.google.cloud.BaseWriteChannel#DEFAULT_CHUNK_SIZE
      */
-    static final int SDK_DEFAULT_CHUNK_SIZE = Math.toIntExact(ByteSizeValue.ofMb(16).getBytes());
+    static final int SDK_DEFAULT_CHUNK_SIZE = 60 * 256 * 1024;
 
     private static final Logger logger = LogManager.getLogger(GoogleCloudStorageBlobStore.class);
 
@@ -652,17 +652,7 @@ private static final class WritableBlobChannel implements WritableByteChannel {
         @SuppressForbidden(reason = "channel is based on a socket")
         @Override
         public int write(final ByteBuffer src) throws IOException {
-            try {
-                return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
-            } catch (IOException e) {
-                // BaseStorageWriteChannel#write wraps StorageException in an IOException, but BaseStorageWriteChannel#close
-                // does not, if we unwrap StorageExceptions here, it simplifies our retry-on-gone logic
-                final StorageException storageException = (StorageException) ExceptionsHelper.unwrap(e, StorageException.class);
-                if (storageException != null) {
-                    throw storageException;
-                }
-                throw e;
-            }
+            return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
         }
 
         @Override

modules/repository-gcs/licenses/opentelemetry-api-LICENSE.txt renamed to modules/repository-gcs/licenses/commons-logging-LICENSE.txt

@@ -41,7 +41,6 @@
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.Proxy;
-import java.net.SocketException;
 import java.net.URI;
 import java.net.URL;
 import java.net.UnknownHostException;
@@ -276,10 +275,6 @@ protected StorageRetryStrategy getRetryStrategy() {
                 if (ExceptionsHelper.unwrap(prevThrowable, UnknownHostException.class) != null) {
                     return true;
                 }
-                // Also retry on `SocketException`s
-                if (ExceptionsHelper.unwrap(prevThrowable, SocketException.class) != null) {
-                    return true;
-                }
                 return delegate.shouldRetry(prevThrowable, prevResponse);
             }
         );