* search: update to ES 9.0.0

Signed-off-by: neo <[email protected]>

Author: neowu

CHANGELOG.md

@@ -4,7 +4,10 @@
 
 * java: update to 24
   > refer to https://openjdk.org/jeps/491
-* search: remove opentelemetry dependency from es lib
+* search: update to ES 9.0.0
+  > migrated to apache httpclient 5   
+  > simplify logging
+  > remove opentelemetry dependency
 
 ### 9.1.8 (3/7/2025 - 4/24/2025)
 

build.gradle.kts

@@ -8,10 +8,18 @@ apply(plugin = "project")
 subprojects {
     group = "core.framework"
     version = "9.2.0"
+    repositories {
+        maven {
+            url = uri("https://neowu.github.io/maven-repo/")
+            content {
+                includeGroup("core.framework.elasticsearch.module")
+            }
+        }
+    }
 }
 
-val elasticVersion = "8.15.0"
-val jacksonVersion = "2.17.2"
+val elasticVersion = "9.0.0"
+val jacksonVersion = "2.18.3"
 val junitVersion = "5.12.2"
 val mockitoVersion = "5.17.0"
 val assertjVersion = "3.27.3"
@@ -82,6 +90,7 @@ project("core-ng-search") {
         api("co.elastic.clients:elasticsearch-java:${elasticVersion}") {
             exclude(group = "io.opentelemetry")
         }
+        implementation("commons-logging:commons-logging:1.3.5")
         implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
         testImplementation(project(":core-ng-test"))
     }
@@ -96,10 +105,12 @@ project("core-ng-search-test") {
             exclude(group = "io.opentelemetry")
         }
         implementation("org.elasticsearch.plugin:transport-netty4:${elasticVersion}")
-        implementation("org.codelibs.elasticsearch.module:mapper-extras:${elasticVersion}")         // used by elasticsearch scaled_float
-        implementation("org.codelibs.elasticsearch.module:lang-painless:${elasticVersion}")
-        implementation("org.codelibs.elasticsearch.module:analysis-common:${elasticVersion}")       // used by elasticsearch stemmer
-        implementation("org.codelibs.elasticsearch.module:reindex:${elasticVersion}@jar")           // used by elasticsearch deleteByQuery
+        implementation("core.framework.elasticsearch.module:mapper-extras:${elasticVersion}")       // used by elasticsearch scaled_float
+        implementation("core.framework.elasticsearch.module:lang-painless:${elasticVersion}")
+        implementation("core.framework.elasticsearch.module:analysis-common:${elasticVersion}")     // used by elasticsearch stemmer
+        implementation("core.framework.elasticsearch.module:reindex:${elasticVersion}")             // used by elasticsearch deleteByQuery
+        implementation("org.apache.httpcomponents:httpasyncclient:4.1.5")                           // used by reindex
+        runtimeOnly("org.apache.logging.log4j:log4j-to-slf4j:2.24.3")
     }
 }
 

buildSrc/src/main/kotlin/project.gradle.kts

@@ -25,12 +25,6 @@ subprojects {
 
     repositories {
         mavenCentral()
-        maven {
-            url = uri("https://maven.codelibs.org/")
-            content {
-                includeGroup("org.codelibs.elasticsearch.module")
-            }
-        }
         mavenLocal()
     }
 

core-ng-search-test/src/main/java/core/framework/search/impl/ESLoggerConfigFactory.java

@@ -2,7 +2,7 @@
 
 import core.framework.util.Files;
 import core.framework.util.StopWatch;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;

core-ng-search-test/src/main/java/core/framework/search/impl/LocalElasticSearch.java

@@ -2,12 +2,9 @@
 
 import core.framework.internal.module.ModuleContext;
 import core.framework.internal.module.ShutdownHook;
-import core.framework.search.impl.ESLoggerConfigFactory;
 import core.framework.search.impl.LocalElasticSearch;
-import org.apache.http.HttpHost;
-import org.apache.logging.log4j.core.config.ConfigurationFactory;
-import org.elasticsearch.common.logging.internal.LoggerFactoryImpl;
-import org.elasticsearch.logging.internal.spi.LoggerFactory;
+import org.apache.hc.core5.http.HttpHost;
+import org.elasticsearch.common.logging.LogConfigurator;
 
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -31,7 +28,6 @@ public class TestSearchConfig extends SearchConfig {
     @Override
     protected void initialize(ModuleContext context, String name) {
         super.initialize(context, name);
-        configureLogger();
         startLocalElasticSearch(context);
     }
 
@@ -40,6 +36,8 @@ private void startLocalElasticSearch(ModuleContext context) {
         try {
             // in test env, config is initialized in order and within same thread, so no threading issue
             if (localESHost == null) {
+                LogConfigurator.configureESLogging();
+
                 var server = new LocalElasticSearch();
                 localESHost = server.start();
                 context.shutdownHook.add(ShutdownHook.STAGE_6, timeout -> server.close());
@@ -56,27 +54,5 @@ public void host(String host) {
 
     @Override
     public void auth(String apiKeyId, String apiKeySecret) {
-
-    }
-
-    // ES uses log4j2 core api directly, cannot use log4j-to-slf4j to bridge, refer to following exception
-    // here is to bridge to core-ng logger
-    /*
-      java.lang.ClassCastException: class org.apache.logging.slf4j.SLF4JLoggerContext cannot be cast to class org.apache.logging.log4j.core.LoggerContext (org.apache.logging.slf4j.SLF4JLoggerContext and org.apache.logging.log4j.core.LoggerContext are in unnamed module of loader 'app')
-        at org.apache.logging.log4j.core.LoggerContext.getContext(LoggerContext.java:231)
-        at org.apache.logging.log4j.core.config.Configurator.setLevel(Configurator.java:366)
-        at org.elasticsearch.common.logging.Loggers.setLevel(Loggers.java:114)
-        at org.elasticsearch.index.SearchSlowLog.<init>(SearchSlowLog.java:111)
-        at org.elasticsearch.index.IndexModule.<init>(IndexModule.java:176)
-    * */
-    void configureLogger() {
-        // ES starts to migrate log4j api to its own logger api, once important info migrated (like node info), we will remove log4j adapter
-        if (LoggerFactory.provider() == null) LoggerFactory.setInstance(new LoggerFactoryImpl());
-
-        if (System.getProperty(ConfigurationFactory.CONFIGURATION_FACTORY_PROPERTY) != null) return;
-        System.setProperty(ConfigurationFactory.CONFIGURATION_FACTORY_PROPERTY, ESLoggerConfigFactory.class.getName());
-        // refer to org.apache.logging.log4j.core.jmx.Server.PROPERTY_DISABLE_JMX, disable to reduce overhead
-        System.setProperty("log4j2.disable.jmx", "true");
-        ESLoggerConfigFactory.configureLogger();
     }
 }

core-ng-search-test/src/main/java/core/framework/search/module/TestSearchConfig.java

@@ -6,21 +6,23 @@
 import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.painless.PainlessPlugin;
 import org.elasticsearch.plugins.LocalPluginsService;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.transport.netty4.Netty4Plugin;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
  * @author neo
  */
 public class LocalNode extends Node {
     public LocalNode(Settings settings) {
-        super(NodeConstruction.prepareConstruction(new Environment(settings, null), new NodeServiceProvider() {
+        super(NodeConstruction.prepareConstruction(new Environment(settings, null), PluginsLoader.createPluginsLoader(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), false), new NodeServiceProvider() {
             @Override
-            PluginsService newPluginService(Environment environment, Settings settings) {
-                return new LocalPluginsService(settings,
+            PluginsService newPluginService(Environment initialEnvironment, PluginsLoader pluginsLoader) {
+                return new LocalPluginsService(settings, pluginsLoader,
                     List.of(Netty4Plugin.class,           // for http transport
                         MapperExtrasPlugin.class,         // for scaled_float type
                         PainlessPlugin.class,

core-ng-search-test/src/main/java/org/elasticsearch/node/LocalNode.java

@@ -2,12 +2,10 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.jdk.ModuleQualifiedExportsService;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 
 /**
  * @author neo
@@ -18,8 +16,8 @@ public class LocalPluginsService extends PluginsService {
     // refer to https://github.com/elastic/elasticsearch/blob/main/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java
     // refer to https://github.com/elastic/elasticsearch/blob/main/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
     // refer to https://github.com/elastic/elasticsearch/blob/main/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
-    public LocalPluginsService(Settings settings, Collection<Class<? extends Plugin>> pluginClasses) {
-        super(settings, null, null, null);
+    public LocalPluginsService(Settings settings, PluginsLoader loader, Collection<Class<? extends Plugin>> pluginClasses) {
+        super(settings, null, loader);
         plugins = new ArrayList<>(pluginClasses.size());
         for (Class<? extends Plugin> pluginClass : pluginClasses) {
             Plugin plugin = loadPlugin(pluginClass, settings, null);
@@ -28,15 +26,15 @@ public LocalPluginsService(Settings settings, Collection<Class<? extends Plugin>
                 "classpath plugin",
                 "NA",
                 Version.CURRENT.toString(),
-                "21",
+                "24",
                 pluginClass.getName(),
                 null,
                 List.of(),
                 false,
                 false,
                 false,
                 false);
-            plugins.add(new LoadedPlugin(descriptor, plugin, Thread.currentThread().getContextClassLoader(), ModuleLayer.boot()));
+            plugins.add(new LoadedPlugin(descriptor, plugin, Thread.currentThread().getContextClassLoader()));
         }
         loadExtensions(plugins);
     }
@@ -45,8 +43,4 @@ public LocalPluginsService(Settings settings, Collection<Class<? extends Plugin>
     protected final List<LoadedPlugin> plugins() {
         return this.plugins;
     }
-
-    @Override
-    protected void addServerExportsService(Map<String, List<ModuleQualifiedExportsService>> qualifiedExports) {
-    }
 }

core-ng-search-test/src/main/java/org/elasticsearch/plugins/LocalPluginsService.java

@@ -15,6 +15,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.ZonedDateTime;
 import java.util.List;
@@ -27,6 +29,7 @@
  */
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class ElasticSearchAggregationIntegrationTest extends IntegrationTest {
+    private static final Logger log = LoggerFactory.getLogger(ElasticSearchAggregationIntegrationTest.class);
     @Inject
     ElasticSearch elasticSearch;
     @Inject
@@ -62,7 +65,7 @@ void aggregate() {
         assertThat(response.hits).hasSize(1);
         assertThat(response.aggregations).containsKeys("total_value");
 
-        int sum = (int) response.aggregations.get("total_value").sum().value();
+        int sum = response.aggregations.get("total_value").sum().value().intValue();
         assertThat(sum).isEqualTo(21);
     }
 

core-ng-search-test/src/test/java/core/framework/search/impl/ElasticSearchAggregationIntegrationTest.java

@@ -124,7 +124,7 @@ void search() {
             .filter(term("enum_field", JSON.toEnumValue(TestDocument.TestEnum.VALUE1))));
 
         request.sorts.add(SortOptions.of(builder -> builder.script(s ->
-            s.script(script -> script.source("doc['int_field'].value * 3")).type(ScriptSortType.Number))));
+            s.script(script -> script.source(source -> source.scriptString("doc['int_field'].value * 3"))).type(ScriptSortType.Number))));
 
         SearchResponse<TestDocument> response = documentType.search(request);
 

core-ng-search-test/src/test/java/core/framework/search/impl/ElasticSearchIntegrationTest.java

@@ -3,8 +3,6 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
 /**
  * @author neo
  */
@@ -17,9 +15,7 @@ void createTestSearchConfig() {
     }
 
     @Test
-    void configureLogger() {
-        config.configureLogger();
-
-        assertThat(System.getProperty("log4j.configurationFactory")).isNotNull();
+    void auth() {
+        config.auth("id", "secret");
     }
 }

core-ng-search-test/src/test/java/core/framework/search/module/TestSearchConfigTest.java

@@ -3,7 +3,7 @@
 import core.framework.internal.module.PropertyManager;
 import core.framework.search.impl.ElasticSearchHost;
 import core.framework.search.impl.ElasticSearchImpl;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

core-ng-search/src/main/java/core/framework/search/ElasticSearchMigration.java

@@ -1,7 +1,7 @@
 package core.framework.search.impl;
 
 import core.framework.util.Strings;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 
 /**
  * @author neo
@@ -34,6 +34,6 @@ private static HttpHost host(String value) {
             port = Integer.parseInt(value.substring(portIndex + 1));
             hostEnd = portIndex;
         }
-        return new HttpHost(value.substring(hostStart, hostEnd), port, schema);
+        return new HttpHost(schema, value.substring(hostStart, hostEnd), port);
     }
 }