Skip to content

Commit 7740c53

Browse files
karenyrxrgsriram
karenyrx
authored and
rgsriram
committed
[GRPC] Add terms query support in Search GRPC endpoint (opensearch-project#17888)
Signed-off-by: Karen Xu <[email protected]> Signed-off-by: Karen X <[email protected]> Signed-off-by: Sriram Ganesh <[email protected]>
1 parent 0c18f7e commit 7740c53

File tree

9 files changed

+792
-2
lines changed

9 files changed

+792
-2
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3636
- Add update and delete support in pull-based ingestion ([#17822](https://github.com/opensearch-project/OpenSearch/pull/17822))
3737
- Allow maxPollSize and pollTimeout in IngestionSource to be configurable ([#17863](https://github.com/opensearch-project/OpenSearch/pull/17863))
3838
- [Star Tree] [Search] Add query changes to support unsigned-long in star tree ([#17275](https://github.com/opensearch-project/OpenSearch/pull/17275))
39+
- Add TermsQuery support to Search GRPC endpoint ([#17888](https://github.com/opensearch-project/OpenSearch/pull/17888))
3940

4041
### Changed
4142
- Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912))

‎plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/AbstractQueryBuilderProtoUtils.java

+2
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ public static QueryBuilder parseInnerQueryBuilderProto(QueryContainer queryConta
4242
result = MatchNoneQueryBuilderProtoUtils.fromProto(queryContainer.getMatchNone());
4343
} else if (queryContainer.getTermCount() > 0) {
4444
result = TermQueryBuilderProtoUtils.fromProto(queryContainer.getTermMap());
45+
} else if (queryContainer.hasTerms()) {
46+
result = TermsQueryBuilderProtoUtils.fromProto(queryContainer.getTerms());
4547
}
4648
// TODO add more query types
4749
else {

‎plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/TermsLookupProtoUtils.java

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
package org.opensearch.plugin.transport.grpc.proto.request.search.query;
9+
10+
import org.opensearch.core.xcontent.XContentParser;
11+
import org.opensearch.indices.TermsLookup;
12+
import org.opensearch.protobufs.TermsLookupField;
13+
14+
/**
15+
* Utility class for converting TermsLookup Protocol Buffers to OpenSearch objects.
16+
* This class provides methods to transform Protocol Buffer representations of terms lookups
17+
* into their corresponding OpenSearch TermsLookup implementations for search operations.
18+
*/
19+
public class TermsLookupProtoUtils {
20+
21+
private TermsLookupProtoUtils() {
22+
// Utility class, no instances
23+
}
24+
25+
/**
26+
* Converts a Protocol Buffer TermsLookupField to an OpenSearch TermsLookup object.
27+
* Similar to {@link TermsLookup#parseTermsLookup(XContentParser)}
28+
*
29+
* @param termsLookupFieldProto The Protocol Buffer TermsLookupField object containing index, id, path, and optional routing/store values
30+
* @return A configured TermsLookup instance with the appropriate settings
31+
*/
32+
protected static TermsLookup parseTermsLookup(TermsLookupField termsLookupFieldProto) {
33+
34+
String index = termsLookupFieldProto.getIndex();
35+
String id = termsLookupFieldProto.getId();
36+
String path = termsLookupFieldProto.getPath();
37+
38+
TermsLookup termsLookup = new TermsLookup(index, id, path);
39+
40+
if (termsLookupFieldProto.hasRouting()) {
41+
termsLookup.routing(termsLookupFieldProto.getRouting());
42+
}
43+
44+
if (termsLookupFieldProto.hasStore()) {
45+
termsLookup.store(termsLookupFieldProto.getStore());
46+
}
47+
48+
return termsLookup;
49+
}
50+
}

‎plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/TermsQueryBuilderProtoUtils.java

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
package org.opensearch.plugin.transport.grpc.proto.request.search.query;
9+
10+
import com.google.protobuf.ProtocolStringList;
11+
import org.apache.lucene.util.BytesRef;
12+
import org.opensearch.core.common.bytes.BytesArray;
13+
import org.opensearch.core.xcontent.XContentParser;
14+
import org.opensearch.index.query.AbstractQueryBuilder;
15+
import org.opensearch.index.query.TermsQueryBuilder;
16+
import org.opensearch.indices.TermsLookup;
17+
import org.opensearch.protobufs.TermsLookupField;
18+
import org.opensearch.protobufs.TermsLookupFieldStringArrayMap;
19+
import org.opensearch.protobufs.TermsQueryField;
20+
import org.opensearch.protobufs.ValueType;
21+
22+
import java.util.ArrayList;
23+
import java.util.Base64;
24+
import java.util.List;
25+
import java.util.Map;
26+
27+
import static org.opensearch.index.query.AbstractQueryBuilder.maybeConvertToBytesRef;
28+
29+
/**
30+
* Utility class for converting TermQuery Protocol Buffers to OpenSearch objects.
31+
* This class provides methods to transform Protocol Buffer representations of term queries
32+
* into their corresponding OpenSearch TermQueryBuilder implementations for search operations.
33+
*/
34+
public class TermsQueryBuilderProtoUtils {
35+
36+
private TermsQueryBuilderProtoUtils() {
37+
// Utility class, no instances
38+
}
39+
40+
/**
41+
* Converts a Protocol Buffer TermQuery map to an OpenSearch TermQueryBuilder.
42+
* Similar to {@link TermsQueryBuilder#fromXContent(XContentParser)}, this method
43+
* parses the Protocol Buffer representation and creates a properly configured
44+
* TermQueryBuilder with the appropriate field name, value, boost, query name,
45+
* and case sensitivity settings.
46+
*
47+
* @param termsQueryProto The map of field names to Protocol Buffer TermsQuery objects
48+
* @return A configured TermQueryBuilder instance
49+
* @throws IllegalArgumentException if the term query map has more than one element,
50+
* if the field value type is not supported, or if the term query field value is not recognized
51+
*/
52+
protected static TermsQueryBuilder fromProto(TermsQueryField termsQueryProto) {
53+
54+
String fieldName = null;
55+
List<Object> values = null;
56+
TermsLookup termsLookup = null;
57+
58+
String queryName = null;
59+
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
60+
String valueTypeStr = TermsQueryBuilder.ValueType.DEFAULT.name();
61+
62+
if (termsQueryProto.hasBoost()) {
63+
boost = termsQueryProto.getBoost();
64+
}
65+
66+
if (termsQueryProto.hasName()) {
67+
queryName = termsQueryProto.getName();
68+
}
69+
70+
// TODO: remove this parameter when backporting to under OS 2.17
71+
if (termsQueryProto.hasValueType()) {
72+
valueTypeStr = parseValueType(termsQueryProto.getValueType()).name();
73+
}
74+
75+
if (termsQueryProto.getTermsLookupFieldStringArrayMapMap().size() > 1) {
76+
throw new IllegalArgumentException("[" + TermsQueryBuilder.NAME + "] query does not support more than one field. ");
77+
}
78+
79+
for (Map.Entry<String, TermsLookupFieldStringArrayMap> entry : termsQueryProto.getTermsLookupFieldStringArrayMapMap().entrySet()) {
80+
fieldName = entry.getKey();
81+
TermsLookupFieldStringArrayMap termsLookupFieldStringArrayMap = entry.getValue();
82+
83+
if (termsLookupFieldStringArrayMap.hasTermsLookupField()) {
84+
TermsLookupField termsLookupField = termsLookupFieldStringArrayMap.getTermsLookupField();
85+
termsLookup = TermsLookupProtoUtils.parseTermsLookup(termsLookupField);
86+
} else if (termsLookupFieldStringArrayMap.hasStringArray()) {
87+
values = parseValues(termsLookupFieldStringArrayMap.getStringArray().getStringArrayList());
88+
} else {
89+
throw new IllegalArgumentException("termsLookupField and stringArray fields cannot both be null");
90+
}
91+
}
92+
93+
TermsQueryBuilder.ValueType valueType = TermsQueryBuilder.ValueType.fromString(valueTypeStr);
94+
95+
if (valueType == TermsQueryBuilder.ValueType.BITMAP) {
96+
if (values != null && values.size() == 1 && values.get(0) instanceof BytesRef) {
97+
values.set(0, new BytesArray(Base64.getDecoder().decode(((BytesRef) values.get(0)).utf8ToString())));
98+
} else if (termsLookup == null) {
99+
throw new IllegalArgumentException(
100+
"Invalid value for bitmap type: Expected a single-element array with a base64 encoded serialized bitmap."
101+
);
102+
}
103+
}
104+
105+
TermsQueryBuilder termsQueryBuilder;
106+
if (values == null) {
107+
termsQueryBuilder = new TermsQueryBuilder(fieldName, termsLookup);
108+
} else if (termsLookup == null) {
109+
termsQueryBuilder = new TermsQueryBuilder(fieldName, values);
110+
} else {
111+
throw new IllegalArgumentException("values and termsLookup cannot both be null");
112+
}
113+
114+
return termsQueryBuilder.boost(boost).queryName(queryName).valueType(valueType);
115+
}
116+
117+
/**
118+
* Parses a protobuf ScriptLanguage to a String representation
119+
*
120+
* See {@link org.opensearch.index.query.TermsQueryBuilder.ValueType#fromString(String)} }
121+
* *
122+
* @param valueType the Protocol Buffer ValueType to convert
123+
* @return the string representation of the script language
124+
* @throws UnsupportedOperationException if no language was specified
125+
*/
126+
public static TermsQueryBuilder.ValueType parseValueType(ValueType valueType) {
127+
switch (valueType) {
128+
case VALUE_TYPE_BITMAP:
129+
return TermsQueryBuilder.ValueType.BITMAP;
130+
case VALUE_TYPE_DEFAULT:
131+
return TermsQueryBuilder.ValueType.DEFAULT;
132+
case VALUE_TYPE_UNSPECIFIED:
133+
default:
134+
return TermsQueryBuilder.ValueType.DEFAULT;
135+
}
136+
}
137+
138+
/**
139+
* Similar to {@link TermsQueryBuilder#parseValues(XContentParser)}
140+
* @param termsLookupFieldStringArray
141+
* @return
142+
* @throws IllegalArgumentException
143+
*/
144+
static List<Object> parseValues(ProtocolStringList termsLookupFieldStringArray) throws IllegalArgumentException {
145+
List<Object> values = new ArrayList<>();
146+
147+
for (Object value : termsLookupFieldStringArray) {
148+
Object convertedValue = maybeConvertToBytesRef(value);
149+
if (value == null) {
150+
throw new IllegalArgumentException("No value specified for terms query");
151+
}
152+
values.add(convertedValue);
153+
}
154+
return values;
155+
}
156+
}

‎plugins/transport-grpc/src/test/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/AbstractQueryBuilderProtoUtilsTests.java

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.transport.grpc.proto.request.search.query;
10+
11+
import org.opensearch.index.query.MatchAllQueryBuilder;
12+
import org.opensearch.index.query.MatchNoneQueryBuilder;
13+
import org.opensearch.index.query.QueryBuilder;
14+
import org.opensearch.index.query.TermQueryBuilder;
15+
import org.opensearch.index.query.TermsQueryBuilder;
16+
import org.opensearch.protobufs.FieldValue;
17+
import org.opensearch.protobufs.MatchAllQuery;
18+
import org.opensearch.protobufs.MatchNoneQuery;
19+
import org.opensearch.protobufs.QueryContainer;
20+
import org.opensearch.protobufs.StringArray;
21+
import org.opensearch.protobufs.TermQuery;
22+
import org.opensearch.protobufs.TermsLookupFieldStringArrayMap;
23+
import org.opensearch.protobufs.TermsQueryField;
24+
import org.opensearch.test.OpenSearchTestCase;
25+
26+
import java.util.HashMap;
27+
import java.util.Map;
28+
29+
public class AbstractQueryBuilderProtoUtilsTests extends OpenSearchTestCase {
30+
31+
public void testParseInnerQueryBuilderProtoWithMatchAll() {
32+
// Create a QueryContainer with MatchAllQuery
33+
MatchAllQuery matchAllQuery = MatchAllQuery.newBuilder().build();
34+
QueryContainer queryContainer = QueryContainer.newBuilder().setMatchAll(matchAllQuery).build();
35+
36+
// Call parseInnerQueryBuilderProto
37+
QueryBuilder queryBuilder = AbstractQueryBuilderProtoUtils.parseInnerQueryBuilderProto(queryContainer);
38+
39+
// Verify the result
40+
assertNotNull("QueryBuilder should not be null", queryBuilder);
41+
assertTrue("QueryBuilder should be a MatchAllQueryBuilder", queryBuilder instanceof MatchAllQueryBuilder);
42+
}
43+
44+
public void testParseInnerQueryBuilderProtoWithMatchNone() {
45+
// Create a QueryContainer with MatchNoneQuery
46+
MatchNoneQuery matchNoneQuery = MatchNoneQuery.newBuilder().build();
47+
QueryContainer queryContainer = QueryContainer.newBuilder().setMatchNone(matchNoneQuery).build();
48+
49+
// Call parseInnerQueryBuilderProto
50+
QueryBuilder queryBuilder = AbstractQueryBuilderProtoUtils.parseInnerQueryBuilderProto(queryContainer);
51+
52+
// Verify the result
53+
assertNotNull("QueryBuilder should not be null", queryBuilder);
54+
assertTrue("QueryBuilder should be a MatchNoneQueryBuilder", queryBuilder instanceof MatchNoneQueryBuilder);
55+
}
56+
57+
public void testParseInnerQueryBuilderProtoWithTerm() {
58+
// Create a QueryContainer with Term query
59+
Map<String, TermQuery> termMap = new HashMap<>();
60+
61+
// Create a FieldValue for the term value
62+
FieldValue fieldValue = FieldValue.newBuilder().setStringValue("test-value").build();
63+
64+
// Create a TermQuery with the FieldValue
65+
TermQuery termQuery = TermQuery.newBuilder().setValue(fieldValue).build();
66+
67+
termMap.put("test-field", termQuery);
68+
69+
QueryContainer queryContainer = QueryContainer.newBuilder().putAllTerm(termMap).build();
70+
71+
// Call parseInnerQueryBuilderProto
72+
QueryBuilder queryBuilder = AbstractQueryBuilderProtoUtils.parseInnerQueryBuilderProto(queryContainer);
73+
74+
// Verify the result
75+
assertNotNull("QueryBuilder should not be null", queryBuilder);
76+
assertTrue("QueryBuilder should be a TermQueryBuilder", queryBuilder instanceof TermQueryBuilder);
77+
TermQueryBuilder termQueryBuilder = (TermQueryBuilder) queryBuilder;
78+
assertEquals("Field name should match", "test-field", termQueryBuilder.fieldName());
79+
assertEquals("Value should match", "test-value", termQueryBuilder.value());
80+
}
81+
82+
public void testParseInnerQueryBuilderProtoWithTerms() {
83+
// Create a StringArray for terms values
84+
StringArray stringArray = StringArray.newBuilder().addStringArray("value1").addStringArray("value2").build();
85+
86+
// Create a TermsLookupFieldStringArrayMap
87+
TermsLookupFieldStringArrayMap termsLookupFieldStringArrayMap = TermsLookupFieldStringArrayMap.newBuilder()
88+
.setStringArray(stringArray)
89+
.build();
90+
91+
// Create a map for TermsLookupFieldStringArrayMap
92+
Map<String, TermsLookupFieldStringArrayMap> termsLookupFieldStringArrayMapMap = new HashMap<>();
93+
termsLookupFieldStringArrayMapMap.put("test-field", termsLookupFieldStringArrayMap);
94+
95+
// Create a TermsQueryField
96+
TermsQueryField termsQueryField = TermsQueryField.newBuilder()
97+
.putAllTermsLookupFieldStringArrayMap(termsLookupFieldStringArrayMapMap)
98+
.build();
99+
100+
// Create a QueryContainer with Terms query
101+
QueryContainer queryContainer = QueryContainer.newBuilder().setTerms(termsQueryField).build();
102+
103+
// Call parseInnerQueryBuilderProto
104+
QueryBuilder queryBuilder = AbstractQueryBuilderProtoUtils.parseInnerQueryBuilderProto(queryContainer);
105+
106+
// Verify the result
107+
assertNotNull("QueryBuilder should not be null", queryBuilder);
108+
assertTrue("QueryBuilder should be a TermsQueryBuilder", queryBuilder instanceof TermsQueryBuilder);
109+
TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder;
110+
assertEquals("Field name should match", "test-field", termsQueryBuilder.fieldName());
111+
assertEquals("Values size should match", 2, termsQueryBuilder.values().size());
112+
assertEquals("First value should match", "value1", termsQueryBuilder.values().get(0));
113+
assertEquals("Second value should match", "value2", termsQueryBuilder.values().get(1));
114+
}
115+
116+
public void testParseInnerQueryBuilderProtoWithUnsupportedQueryType() {
117+
// Create an empty QueryContainer (no query type specified)
118+
QueryContainer queryContainer = QueryContainer.newBuilder().build();
119+
120+
// Call parseInnerQueryBuilderProto, should throw UnsupportedOperationException
121+
UnsupportedOperationException exception = expectThrows(
122+
UnsupportedOperationException.class,
123+
() -> AbstractQueryBuilderProtoUtils.parseInnerQueryBuilderProto(queryContainer)
124+
);
125+
126+
// Verify the exception message
127+
assertTrue("Exception message should mention 'not supported yet'", exception.getMessage().contains("not supported yet"));
128+
}
129+
}

0 commit comments

Comments
 (0)