Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Commit 254f2e0

Browse files
joshuali925joshuali925
authored
Opendistro Release 1.9.0 (#532)
* prepare odfe 1.9 * Fix all ES 7.8 compile and build errors * Revert changes as Lombok is working now * Update CustomExternalTestCluster.java * Fix license headers check * Use splitFieldsByMetadata to separate fields when calling SearchHit constructor * More fixes for ODFE 1.9 * Remove todo statement * Add ODFE 1.9.0 release notes
1 parent fb2ed91 commit 254f2e0

30 files changed

+192
-128
lines changed

‎build.gradle

+9-3
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
buildscript {
1717
ext {
18-
es_version = "7.7.0"
18+
es_version = "7.8.0"
1919
}
2020
// This isn't applying from repositories.gradle so repeating it here
2121
repositories {
@@ -40,7 +40,7 @@ repositories {
4040
}
4141

4242
ext {
43-
opendistroVersion = '1.8.0'
43+
opendistroVersion = '1.9.0'
4444
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
4545
}
4646

@@ -97,6 +97,10 @@ licenseHeaders {
9797
excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**']
9898
}
9999

100+
tasks.withType(licenseHeaders.class) {
101+
additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
102+
}
103+
100104
// TODO: need to fix java doc to enable JavaDoc
101105
javadoc.enabled = false
102106
esplugin {
@@ -272,14 +276,16 @@ dependencies {
272276
testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}"
273277

274278
// JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception.
275-
testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0'
279+
testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0'
276280
testCompile group: 'com.h2database', name: 'h2', version: '1.4.200'
277281
testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0'
278282
//testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3'
279283
}
280284

281285
apply plugin: 'nebula.ospackage'
282286

287+
validateNebulaPom.enabled = false
288+
283289
// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
284290
afterEvaluate {
285291
ospackage {

‎gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313
# permissions and limitations under the License.
1414
#
1515

16-
version=1.8.0
16+
version=1.9.0

‎gradle/wrapper/gradle-wrapper.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# permissions and limitations under the License.
1414
#
1515

16-
distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
16+
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
1717
distributionBase=GRADLE_USER_HOME
1818
distributionPath=wrapper/dists
1919
zipStorePath=wrapper/dists

‎release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
## 2020-06-23 Version 1.9.0.0
2+
3+
### Features
4+
#### Elasticsearch Compatibility
5+
* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531))
6+
7+
#### Documentation
8+
* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README
9+
10+
### Enhancements
11+
#### SQL Features
12+
* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster
13+
14+
### Bugfixes
15+
* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488))
16+
* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521))

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) {
109109
this.clusterService = clusterService;
110110

111111
clusterService.addListener(event -> {
112-
if (event.metaDataChanged()) {
112+
if (event.metadataChanged()) {
113113
// State in cluster service is already changed to event.state() before listener fired
114114
if (LOG.isDebugEnabled()) {
115115
LOG.debug("Metadata in cluster state changed: {}",
116-
new IndexMappings(clusterService.state().metaData()));
116+
new IndexMappings(clusterService.state().metadata()));
117117
}
118118
cache.invalidateAll();
119119
}
@@ -169,8 +169,8 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) {
169169
}
170170

171171
/**
172-
* Get field mappings by index expressions, type and field filter. Because IndexMetaData/MappingMetaData
173-
* is hard to convert to FieldMappingMetaData, custom mapping domain objects are being used here. In future,
172+
* Get field mappings by index expressions, type and field filter. Because IndexMetadata/MappingMetadata
173+
* is hard to convert to FieldMappingMetadata, custom mapping domain objects are being used here. In future,
174174
* it should be moved to domain model layer for all ES specific knowledge.
175175
* <p>
176176
* Note that cluster state may be change inside ES so it's possible to read different state in 2 accesses
@@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[
222222
Function<String, Predicate<String>> fieldFilter) throws IOException {
223223
LOG.debug("Cache didn't help. Load and parse mapping in cluster state");
224224
return new IndexMappings(
225-
state.metaData().findMappings(indices, types, fieldFilter)
225+
state.metadata().findMappings(indices, types, fieldFilter)
226226
);
227227
}
228228

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import java.util.Map;
2222

2323
import static java.util.Collections.emptyMap;
24-
import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
24+
import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
2525

2626
/**
2727
* Field mapping that parses native ES mapping.
@@ -39,7 +39,7 @@ public class FieldMapping {
3939
/**
4040
* Native mapping information returned from ES
4141
*/
42-
private final Map<String, FieldMappingMetaData> typeMappings;
42+
private final Map<String, FieldMappingMetadata> typeMappings;
4343

4444
/**
4545
* Maps a field name to Field object that specified in query explicitly
@@ -51,7 +51,7 @@ public FieldMapping(String fieldName) {
5151
}
5252

5353
public FieldMapping(String fieldName,
54-
Map<String, FieldMappingMetaData> typeMappings,
54+
Map<String, FieldMappingMetadata> typeMappings,
5555
Map<String, Field> specifiedFieldByNames) {
5656

5757
this.fieldName = fieldName;
@@ -119,16 +119,16 @@ public String path() {
119119
}
120120

121121
/**
122-
* Used to retrieve the type of fields from metaData map structures for both regular and nested fields
122+
* Used to retrieve the type of fields from metadata map structures for both regular and nested fields
123123
*/
124124
@SuppressWarnings("unchecked")
125125
public String type() {
126-
FieldMappingMetaData metaData = typeMappings.get(fieldName);
127-
Map<String, Object> source = metaData.sourceAsMap();
126+
FieldMappingMetadata metadata = typeMappings.get(fieldName);
127+
Map<String, Object> source = metadata.sourceAsMap();
128128
String[] fieldPath = fieldName.split("\\.");
129129

130130
/*
131-
* When field is not nested the metaData source is fieldName -> type
131+
* When field is not nested the metadata source is fieldName -> type
132132
* When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type
133133
*/
134134
String root = (fieldPath.length == 1) ? fieldName : fieldPath[1];

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
1717

18-
import org.elasticsearch.cluster.metadata.MappingMetaData;
18+
import org.elasticsearch.cluster.metadata.MappingMetadata;
1919
import org.json.JSONObject;
2020

2121
import java.util.HashMap;
@@ -61,7 +61,7 @@ public class FieldMappings implements Mappings<Map<String, Object>> {
6161
*/
6262
private final Map<String, Object> fieldMappings;
6363

64-
public FieldMappings(MappingMetaData mappings) {
64+
public FieldMappings(MappingMetadata mappings) {
6565
fieldMappings = mappings.sourceAsMap();
6666
}
6767

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515

1616
package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
1717

18-
import org.elasticsearch.cluster.metadata.MappingMetaData;
19-
import org.elasticsearch.cluster.metadata.MetaData;
18+
import org.elasticsearch.cluster.metadata.MappingMetadata;
19+
import org.elasticsearch.cluster.metadata.Metadata;
2020
import org.elasticsearch.common.collect.ImmutableOpenMap;
2121

2222
import java.util.Map;
@@ -55,12 +55,12 @@ public IndexMappings() {
5555
this.indexMappings = emptyMap();
5656
}
5757

58-
public IndexMappings(MetaData metaData) {
59-
this.indexMappings = buildMappings(metaData.indices(),
60-
indexMetaData -> new TypeMappings(indexMetaData.getMappings()));
58+
public IndexMappings(Metadata metadata) {
59+
this.indexMappings = buildMappings(metadata.indices(),
60+
indexMetadata -> new TypeMappings(indexMetadata.getMappings()));
6161
}
6262

63-
public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
63+
public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings) {
6464
this.indexMappings = buildMappings(mappings, TypeMappings::new);
6565
}
6666

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
1717

18-
import org.elasticsearch.cluster.metadata.MappingMetaData;
18+
import org.elasticsearch.cluster.metadata.MappingMetadata;
1919
import org.elasticsearch.common.collect.ImmutableOpenMap;
2020

2121
import java.util.Map;
@@ -36,7 +36,7 @@ public class TypeMappings implements Mappings<FieldMappings> {
3636
*/
3737
private final Map<String, FieldMappings> typeMappings;
3838

39-
public TypeMappings(ImmutableOpenMap<String, MappingMetaData> mappings) {
39+
public TypeMappings(ImmutableOpenMap<String, MappingMetadata> mappings) {
4040
typeMappings = buildMappings(mappings, FieldMappings::new);
4141
}
4242

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ private void async(Client client, Map<String, String> params, QueryAction queryA
143143

144144
// Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
145145
threadPool.schedule(
146-
threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
146+
LogUtils.withCurrentContext(runnable),
147147
new TimeValue(0L),
148148
SQL_WORKER_THREAD_POOL_NAME
149149
);

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java

+8-8
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
2020
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
2121
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
22-
import org.elasticsearch.cluster.metadata.AliasMetaData;
23-
import org.elasticsearch.cluster.metadata.MappingMetaData;
22+
import org.elasticsearch.cluster.metadata.AliasMetadata;
23+
import org.elasticsearch.cluster.metadata.MappingMetadata;
2424
import org.elasticsearch.common.collect.ImmutableOpenMap;
2525
import org.elasticsearch.common.settings.Settings;
2626
import org.elasticsearch.common.xcontent.ToXContent;
@@ -77,22 +77,22 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
7777
return new BytesRestResponse(RestStatus.OK, builder);
7878
}
7979

80-
private void writeAliases(List<AliasMetaData> aliases, XContentBuilder builder, ToXContent.Params params)
80+
private void writeAliases(List<AliasMetadata> aliases, XContentBuilder builder, ToXContent.Params params)
8181
throws IOException {
8282
builder.startObject(Fields.ALIASES);
8383
if (aliases != null) {
84-
for (AliasMetaData alias : aliases) {
85-
AliasMetaData.Builder.toXContent(alias, builder, params);
84+
for (AliasMetadata alias : aliases) {
85+
AliasMetadata.Builder.toXContent(alias, builder, params);
8686
}
8787
}
8888
builder.endObject();
8989
}
9090

91-
private void writeMappings(ImmutableOpenMap<String, MappingMetaData> mappings,
91+
private void writeMappings(ImmutableOpenMap<String, MappingMetadata> mappings,
9292
XContentBuilder builder, ToXContent.Params params) throws IOException {
9393
builder.startObject(Fields.MAPPINGS);
9494
if (mappings != null) {
95-
for (ObjectObjectCursor<String, MappingMetaData> typeEntry : mappings) {
95+
for (ObjectObjectCursor<String, MappingMetadata> typeEntry : mappings) {
9696
builder.field(typeEntry.key);
9797
builder.map(typeEntry.value.sourceAsMap());
9898
}
@@ -114,4 +114,4 @@ static class Fields {
114114
static final String SETTINGS = "settings";
115115
static final String WARMERS = "warmers";
116116
}
117-
}
117+
}

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private void async(Client client, Map<String, String> params, RestChannel channe
9393

9494
// Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
9595
threadPool.schedule(
96-
threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
96+
LogUtils.withCurrentContext(runnable),
9797
new TimeValue(0L),
9898
SQL_WORKER_THREAD_POOL_NAME
9999
);

‎src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java

+18-18
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
2323
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
2424
import org.elasticsearch.client.Client;
25-
import org.elasticsearch.cluster.metadata.MappingMetaData;
25+
import org.elasticsearch.cluster.metadata.MappingMetadata;
2626
import org.elasticsearch.common.collect.ImmutableOpenMap;
2727

2828
import java.util.ArrayList;
@@ -89,33 +89,33 @@ private List<Column> loadColumns() {
8989
private List<Row> loadRows() {
9090
List<Row> rows = new ArrayList<>();
9191
GetIndexResponse indexResponse = (GetIndexResponse) queryResult;
92-
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = indexResponse.getMappings();
92+
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> indexMappings = indexResponse.getMappings();
9393

9494
// Iterate through indices in indexMappings
95-
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexCursor : indexMappings) {
95+
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetadata>> indexCursor : indexMappings) {
9696
String index = indexCursor.key;
9797

9898
// Check to see if index matches given pattern
9999
if (matchesPattern(index, statement.getIndexPattern())) {
100-
ImmutableOpenMap<String, MappingMetaData> typeMapping = indexCursor.value;
100+
ImmutableOpenMap<String, MappingMetadata> typeMapping = indexCursor.value;
101101
// Assuming ES 6.x, iterate through the only type of the index to get mapping data
102-
for (ObjectObjectCursor<String, MappingMetaData> typeCursor : typeMapping) {
103-
MappingMetaData mappingMetaData = typeCursor.value;
102+
for (ObjectObjectCursor<String, MappingMetadata> typeCursor : typeMapping) {
103+
MappingMetadata mappingMetadata = typeCursor.value;
104104
// Load rows for each field in the mapping
105-
rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap()));
105+
rows.addAll(loadIndexData(index, mappingMetadata.getSourceAsMap()));
106106
}
107107
}
108108
}
109109
return rows;
110110
}
111111

112112
@SuppressWarnings("unchecked")
113-
private List<Row> loadIndexData(String index, Map<String, Object> mappingMetaData) {
113+
private List<Row> loadIndexData(String index, Map<String, Object> mappingMetadata) {
114114
List<Row> rows = new ArrayList<>();
115115

116-
Map<String, String> flattenedMetaData = flattenMappingMetaData(mappingMetaData, "", new HashMap<>());
116+
Map<String, String> flattenedMetadata = flattenMappingMetadata(mappingMetadata, "", new HashMap<>());
117117
int position = 1; // Used as an arbitrary ORDINAL_POSITION value for the time being
118-
for (Entry<String, String> entry : flattenedMetaData.entrySet()) {
118+
for (Entry<String, String> entry : flattenedMetadata.entrySet()) {
119119
String columnPattern = statement.getColumnPattern();
120120

121121
// Check to see if column name matches pattern, if given
@@ -153,21 +153,21 @@ private Map<String, Object> loadRowData(String index, String column, String type
153153
* 'GetIndexRequestBuilder' that was used in the old ShowQueryAction. Since the format of the resulting meta data
154154
* is different, this method is being used to flatten and retrieve types.
155155
* <p>
156-
* In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetaData
157-
* whereas here we are using MappingMetaData.
156+
* In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetadata
157+
* whereas here we are using MappingMetadata.
158158
*/
159159
@SuppressWarnings("unchecked")
160-
private Map<String, String> flattenMappingMetaData(Map<String, Object> mappingMetaData,
160+
private Map<String, String> flattenMappingMetadata(Map<String, Object> mappingMetadata,
161161
String currPath,
162162
Map<String, String> flattenedMapping) {
163-
Map<String, Object> properties = (Map<String, Object>) mappingMetaData.get("properties");
163+
Map<String, Object> properties = (Map<String, Object>) mappingMetadata.get("properties");
164164
for (Entry<String, Object> entry : properties.entrySet()) {
165-
Map<String, Object> metaData = (Map<String, Object>) entry.getValue();
165+
Map<String, Object> metadata = (Map<String, Object>) entry.getValue();
166166

167167
String fullPath = addToPath(currPath, entry.getKey());
168-
flattenedMapping.put(fullPath, (String) metaData.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
169-
if (metaData.containsKey("properties")) {
170-
flattenedMapping = flattenMappingMetaData(metaData, fullPath, flattenedMapping);
168+
flattenedMapping.put(fullPath, (String) metadata.getOrDefault("type", DEFAULT_OBJECT_DATATYPE));
169+
if (metadata.containsKey("properties")) {
170+
flattenedMapping = flattenMappingMetadata(metadata, fullPath, flattenedMapping);
171171
}
172172
}
173173

0 commit comments

Comments
 (0)