Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Commit 50ce34f

Browse files
penghuopenghuo
authored
SQL release for Elasticsearch 7.10 (#834)
1 parent 33de457 commit 50ce34f

File tree

11 files changed

+128
-103
lines changed

11 files changed

+128
-103
lines changed

‎build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
buildscript {
1717
ext {
18-
es_version = "7.9.1"
18+
es_version = "7.10.0"
1919
}
2020

2121
repositories {
@@ -43,7 +43,7 @@ repositories {
4343
}
4444

4545
ext {
46-
opendistroVersion = '1.11.0'
46+
opendistroVersion = '1.12.0'
4747
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
4848
}
4949

‎doctest/build.gradle

+18-3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
1+
import org.elasticsearch.gradle.testclusters.RunTask
2+
13
plugins {
24
id "com.wiredforcode.spawn" version "0.8.2"
35
id 'base'
46
}
57

8+
apply plugin: 'elasticsearch.testclusters'
9+
610
def path = project(':').projectDir
711
// temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from
812
// plugin module, and will only build ppl in it.
9-
def plugin_path = project(':plugin').projectDir
13+
def plugin_path = project(':doctest').projectDir
1014

1115
task bootstrap(type: Exec) {
1216
inputs.file "$projectDir/bootstrap.sh"
@@ -17,7 +21,7 @@ task bootstrap(type: Exec) {
1721

1822
//evaluationDependsOn(':')
1923
task startES(type: SpawnProcessTask) {
20-
command "${path}/gradlew -p ${plugin_path} run"
24+
command "${path}/gradlew -p ${plugin_path} runRestTestCluster"
2125
ready 'started'
2226
}
2327

@@ -36,4 +40,15 @@ doctest.dependsOn startES
3640
doctest.finalizedBy stopES
3741

3842
build.dependsOn doctest
39-
clean.dependsOn(cleanBootstrap)
43+
clean.dependsOn(cleanBootstrap)
44+
45+
testClusters {
46+
docTestCluster {
47+
plugin ':plugin'
48+
testDistribution = 'oss'
49+
}
50+
}
51+
tasks.register("runRestTestCluster", RunTask) {
52+
description = 'Runs elasticsearch ODFE SQL plugin'
53+
useCluster testClusters.docTestCluster;
54+
}

‎elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java

+11-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import java.util.Map;
2727
import java.util.function.Function;
2828
import java.util.function.Predicate;
29-
import lombok.RequiredArgsConstructor;
3029
import org.apache.logging.log4j.ThreadContext;
3130
import org.elasticsearch.action.support.IndicesOptions;
3231
import org.elasticsearch.client.node.NodeClient;
@@ -39,7 +38,6 @@
3938
import org.elasticsearch.threadpool.ThreadPool;
4039

4140
/** Elasticsearch connection by node client. */
42-
@RequiredArgsConstructor
4341
public class ElasticsearchNodeClient implements ElasticsearchClient {
4442

4543
/** Default types and field filter to match all. */
@@ -55,10 +53,20 @@ public class ElasticsearchNodeClient implements ElasticsearchClient {
5553
private final NodeClient client;
5654

5755
/** Index name expression resolver to get concrete index name. */
58-
private final IndexNameExpressionResolver resolver = new IndexNameExpressionResolver();
56+
private final IndexNameExpressionResolver resolver;
5957

6058
private static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker";
6159

60+
/**
61+
* Constructor of ElasticsearchNodeClient.
62+
*/
63+
public ElasticsearchNodeClient(ClusterService clusterService,
64+
NodeClient client) {
65+
this.clusterService = clusterService;
66+
this.client = client;
67+
this.resolver = new IndexNameExpressionResolver(client.threadPool().getThreadContext());
68+
}
69+
6270
/**
6371
* Get field mappings of index by an index expression. Majority is copied from legacy
6472
* LocalClusterState.

‎elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.elasticsearch.cluster.metadata.Metadata;
5656
import org.elasticsearch.cluster.service.ClusterService;
5757
import org.elasticsearch.common.collect.ImmutableOpenMap;
58+
import org.elasticsearch.common.util.concurrent.ThreadContext;
5859
import org.elasticsearch.common.xcontent.DeprecationHandler;
5960
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
6061
import org.elasticsearch.common.xcontent.XContentParser;
@@ -84,6 +85,9 @@ class ElasticsearchNodeClientTest {
8485
@Mock
8586
private SearchHit searchHit;
8687

88+
@Mock
89+
private ThreadContext threadContext;
90+
8791
private ExprTupleValue exprTupleValue = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id",
8892
new ExprIntegerValue(1)));
8993

@@ -198,6 +202,7 @@ public void search() {
198202
void schedule() {
199203
ThreadPool threadPool = mock(ThreadPool.class);
200204
when(nodeClient.threadPool()).thenReturn(threadPool);
205+
when(threadPool.getThreadContext()).thenReturn(threadContext);
201206

202207
doAnswer(
203208
invocation -> {

‎gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313
# permissions and limitations under the License.
1414
#
1515

16-
version=1.11.0
16+
version=1.12.0

‎gradle/wrapper/gradle-wrapper.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# permissions and limitations under the License.
1414
#
1515

16-
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
16+
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip
1717
distributionBase=GRADLE_USER_HOME
1818
distributionPath=wrapper/dists
1919
zipStorePath=wrapper/dists

‎integ-test/build.gradle

+72-87
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ tasks.withType(licenseHeaders.class) {
1717
}
1818

1919
validateNebulaPom.enabled = false
20+
loggerUsageCheck.enabled = false
2021

2122
repositories {
2223
mavenCentral()
@@ -63,14 +64,15 @@ compileTestJava {
6364
}
6465
}
6566

66-
tasks.integTest.dependsOn(':plugin:bundlePlugin', ':integ-test:integTestWithNewEngine')
67-
testClusters.integTest {
67+
testClusters.all {
6868
testDistribution = 'oss'
69-
plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile)
69+
plugin ":plugin"
7070
}
7171

7272
// Run only legacy SQL ITs with new SQL engine disabled
73-
integTest.runner {
73+
integTest {
74+
dependsOn (':plugin:bundlePlugin',':integ-test:integTestWithNewEngine')
75+
7476
systemProperty 'tests.security.manager', 'false'
7577
systemProperty('project.root', project.projectDir.absolutePath)
7678

@@ -95,112 +97,95 @@ integTest.runner {
9597
// Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled
9698
task integTestWithNewEngine(type: RestIntegTestTask) {
9799
dependsOn ':plugin:bundlePlugin'
98-
runner {
99-
systemProperty 'tests.security.manager', 'false'
100-
systemProperty('project.root', project.projectDir.absolutePath)
101100

102-
systemProperty "https", System.getProperty("https")
103-
systemProperty "user", System.getProperty("user")
104-
systemProperty "password", System.getProperty("password")
101+
systemProperty 'tests.security.manager', 'false'
102+
systemProperty('project.root', project.projectDir.absolutePath)
105103

106-
// Enable new SQL engine
107-
systemProperty 'enableNewEngine', 'true'
104+
systemProperty "https", System.getProperty("https")
105+
systemProperty "user", System.getProperty("user")
106+
systemProperty "password", System.getProperty("password")
108107

109-
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
110-
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
111-
doFirst { systemProperty 'cluster.debug', getDebug() }
108+
// Enable new SQL engine
109+
systemProperty 'enableNewEngine', 'true'
112110

113-
if (System.getProperty("test.debug") != null) {
114-
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
115-
}
111+
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
112+
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
113+
doFirst { systemProperty 'cluster.debug', getDebug() }
116114

117-
exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
118-
exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**'
115+
if (System.getProperty("test.debug") != null) {
116+
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
117+
}
119118

120-
// Explain IT is dependent on internal implementation of old engine so it's not necessary
121-
// to run these with new engine and not necessary to make this consistent with old engine.
122-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/ExplainIT.class'
123-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatterIT.class'
124-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/TermQueryExplainIT.class'
119+
exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
120+
exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**'
125121

126-
// Skip old semantic analyzer IT because analyzer in new engine has different behavior
127-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.class'
122+
// Explain IT is dependent on internal implementation of old engine so it's not necessary
123+
// to run these with new engine and not necessary to make this consistent with old engine.
124+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/ExplainIT.class'
125+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatterIT.class'
126+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/TermQueryExplainIT.class'
128127

129-
// Skip this IT to avoid breaking tests due to inconsistency in JDBC schema
130-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.class'
128+
// Skip old semantic analyzer IT because analyzer in new engine has different behavior
129+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.class'
131130

132-
// Skip this IT because all assertions are against explain output
133-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.class'
134-
}
135-
}
131+
// Skip this IT to avoid breaking tests due to inconsistency in JDBC schema
132+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.class'
136133

137-
testClusters.integTestWithNewEngine {
138-
testDistribution = 'oss'
139-
plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile)
134+
// Skip this IT because all assertions are against explain output
135+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.class'
140136
}
141137

142138

139+
140+
143141
task docTest(type: RestIntegTestTask) {
144142
dependsOn ':plugin:bundlePlugin'
145-
runner {
146-
systemProperty 'tests.security.manager', 'false'
147-
systemProperty('project.root', project.projectDir.absolutePath)
148-
149-
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
150-
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
151-
doFirst { systemProperty 'cluster.debug', getDebug()}
152-
153-
if (System.getProperty("test.debug") != null) {
154-
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
155-
}
156-
157-
include 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
158-
exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class'
159-
exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class'
160-
exclude 'com/amazon/opendistroforelasticsearch/sql/sql/**/*IT.class'
161-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class'
143+
144+
systemProperty 'tests.security.manager', 'false'
145+
systemProperty('project.root', project.projectDir.absolutePath)
146+
147+
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
148+
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
149+
doFirst { systemProperty 'cluster.debug', getDebug()}
150+
151+
if (System.getProperty("test.debug") != null) {
152+
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
162153
}
163-
}
164154

165-
testClusters.docTest {
166-
testDistribution = 'oss'
167-
plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile)
155+
include 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
156+
exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class'
157+
exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class'
158+
exclude 'com/amazon/opendistroforelasticsearch/sql/sql/**/*IT.class'
159+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class'
168160
}
169161

170-
171162
task comparisonTest(type: RestIntegTestTask) {
172163
dependsOn ':plugin:bundlePlugin'
173-
runner {
174-
systemProperty 'tests.security.manager', 'false'
175-
systemProperty('project.root', project.projectDir.absolutePath)
176-
177-
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
178-
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
179-
doFirst { systemProperty 'cluster.debug', getDebug()}
180-
181-
if (System.getProperty("test.debug") != null) {
182-
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
183-
}
184-
185-
include 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class'
186-
exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
187-
exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class'
188-
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class'
189-
190-
// Enable logging output to console
191-
testLogging.showStandardStreams true
192-
193-
// Pass down system properties to IT class
194-
systemProperty "esHost", System.getProperty("esHost")
195-
systemProperty "dbUrl", System.getProperty("dbUrl")
196-
systemProperty "otherDbUrls", System.getProperty("otherDbUrls")
197-
systemProperty "queries", System.getProperty("queries")
164+
165+
systemProperty 'tests.security.manager', 'false'
166+
systemProperty('project.root', project.projectDir.absolutePath)
167+
168+
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
169+
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
170+
doFirst { systemProperty 'cluster.debug', getDebug()}
171+
172+
if (System.getProperty("test.debug") != null) {
173+
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
198174
}
199-
}
200175

201-
testClusters.comparisonTest {
202-
testDistribution = 'oss'
203-
plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile)
176+
include 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class'
177+
exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class'
178+
exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class'
179+
exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class'
180+
181+
// Enable logging output to console
182+
testLogging.showStandardStreams true
183+
184+
// Pass down system properties to IT class
185+
systemProperty "esHost", System.getProperty("esHost")
186+
systemProperty "dbUrl", System.getProperty("dbUrl")
187+
systemProperty "otherDbUrls", System.getProperty("otherDbUrls")
188+
systemProperty "queries", System.getProperty("queries")
204189
}
205190

206191
task compileJdbc(type:Exec) {

‎legacy/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ dependencies {
8989

9090
testCompile group: 'org.hamcrest', name: 'hamcrest-core', version:'2.2'
9191
// testCompile group: 'com.alibaba', name: 'fastjson', version:'1.2.56'
92-
testCompile group: 'org.mockito', name: 'mockito-core', version:'2.23.4'
92+
testCompile group: 'org.mockito', name: 'mockito-core', version:'3.5.0'
9393
testCompile group: 'junit', name: 'junit', version: '4.12'
9494
testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}"
9595

‎legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/AggMaker.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@
4848
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
4949
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
5050
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
51-
import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
5251
import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
52+
import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds;
5353
import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
5454
import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder;
5555
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
@@ -586,7 +586,7 @@ private DateHistogramAggregationBuilder dateHistogram(MethodField field) throws
586586
case "extended_bounds":
587587
String[] bounds = value.split(":");
588588
if (bounds.length == 2) {
589-
dateHistogram.extendedBounds(new ExtendedBounds(bounds[0], bounds[1]));
589+
dateHistogram.extendedBounds(new LongBounds(bounds[0], bounds[1]));
590590
}
591591
break;
592592

0 commit comments

Comments
 (0)