Skip to content

Commit 1ae7561

Browse files
bharath-techiebharath-techie
committed
Supporting object fields in star-tree index
Signed-off-by: bharath-techie <[email protected]>
1 parent 3da97f2 commit 1ae7561

File tree

5 files changed

+387
-6
lines changed

5 files changed

+387
-6
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

Lines changed: 240 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import org.opensearch.index.compositeindex.CompositeIndexSettings;
2626
import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit;
2727
import org.opensearch.index.compositeindex.datacube.DateDimension;
28+
import org.opensearch.index.compositeindex.datacube.KeywordDimension;
2829
import org.opensearch.index.compositeindex.datacube.MetricStat;
30+
import org.opensearch.index.compositeindex.datacube.NumericDimension;
2931
import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
3032
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
3133
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter;
@@ -41,6 +43,7 @@
4143
import java.util.Arrays;
4244
import java.util.Collections;
4345
import java.util.List;
46+
import java.util.Map;
4447
import java.util.Set;
4548

4649
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -113,6 +116,95 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
113116
}
114117
}
115118

119+
private static XContentBuilder createNestedTestMapping() {
120+
try {
121+
return jsonBuilder().startObject()
122+
.startObject("composite")
123+
.startObject("startree-1")
124+
.field("type", "star_tree")
125+
.startObject("config")
126+
.startObject("date_dimension")
127+
.field("name", "timestamp")
128+
.endObject()
129+
.startArray("ordered_dimensions")
130+
.startObject()
131+
.field("name", "nested.nested1.status")
132+
.endObject()
133+
.startObject()
134+
.field("name", "nested.nested1.keyword_dv")
135+
.endObject()
136+
.endArray()
137+
.startArray("metrics")
138+
.startObject()
139+
.field("name", "nested3.numeric_dv")
140+
.endObject()
141+
.endArray()
142+
.endObject()
143+
.endObject()
144+
.endObject()
145+
.startObject("properties")
146+
.startObject("timestamp")
147+
.field("type", "date")
148+
.endObject()
149+
.startObject("nested3")
150+
.startObject("properties")
151+
.startObject("numeric_dv")
152+
.field("type", "integer")
153+
.field("doc_values", true)
154+
.endObject()
155+
.endObject()
156+
.endObject()
157+
.startObject("numeric")
158+
.field("type", "integer")
159+
.field("doc_values", false)
160+
.endObject()
161+
.startObject("nested")
162+
.startObject("properties")
163+
.startObject("nested1")
164+
.startObject("properties")
165+
.startObject("status")
166+
.field("type", "integer")
167+
.field("doc_values", true)
168+
.endObject()
169+
.startObject("keyword_dv")
170+
.field("type", "keyword")
171+
.field("doc_values", true)
172+
.endObject()
173+
.endObject()
174+
.endObject()
175+
.endObject()
176+
.endObject()
177+
.startObject("nested-not-startree")
178+
.startObject("properties")
179+
.startObject("nested1")
180+
.startObject("properties")
181+
.startObject("status")
182+
.field("type", "integer")
183+
.field("doc_values", true)
184+
.endObject()
185+
.startObject("keyword_dv")
186+
.field("type", "keyword")
187+
.field("doc_values", true)
188+
.endObject()
189+
.endObject()
190+
.endObject()
191+
.endObject()
192+
.endObject()
193+
.startObject("keyword")
194+
.field("type", "keyword")
195+
.field("doc_values", false)
196+
.endObject()
197+
.startObject("ip")
198+
.field("type", "ip")
199+
.field("doc_values", false)
200+
.endObject()
201+
.endObject()
202+
.endObject();
203+
} catch (IOException e) {
204+
throw new IllegalStateException(e);
205+
}
206+
}
207+
116208
private static XContentBuilder createDateTestMapping(boolean duplicate) {
117209
try {
118210
return jsonBuilder().startObject()
@@ -467,6 +559,46 @@ public void testValidCompositeIndexWithDates() {
467559
}
468560
}
469561

562+
public void testValidCompositeIndexWithNestedFields() {
563+
prepareCreate(TEST_INDEX).setMapping(createNestedTestMapping()).setSettings(settings).get();
564+
Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
565+
for (IndicesService service : dataNodeInstances) {
566+
final Index index = resolveIndex("test");
567+
if (service.hasIndex(index)) {
568+
IndexService indexService = service.indexService(index);
569+
Set<CompositeMappedFieldType> fts = indexService.mapperService().getCompositeFieldTypes();
570+
571+
for (CompositeMappedFieldType ft : fts) {
572+
assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
573+
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
574+
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
575+
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
576+
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
577+
List<DateTimeUnitRounding> expectedTimeUnits = Arrays.asList(
578+
new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR),
579+
DataCubeDateTimeUnit.HALF_HOUR_OF_DAY
580+
);
581+
for (int i = 0; i < dateDim.getIntervals().size(); i++) {
582+
assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName());
583+
}
584+
assertEquals("nested.nested1.status", starTreeFieldType.getDimensions().get(1).getField());
585+
assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension);
586+
assertEquals("nested.nested1.keyword_dv", starTreeFieldType.getDimensions().get(2).getField());
587+
assertTrue(starTreeFieldType.getDimensions().get(2) instanceof KeywordDimension);
588+
assertEquals("nested3.numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
589+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
590+
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
591+
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
592+
assertEquals(
593+
StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
594+
starTreeFieldType.getStarTreeConfig().getBuildMode()
595+
);
596+
assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
597+
}
598+
}
599+
}
600+
}
601+
470602
public void testValidCompositeIndexWithDuplicateDates() {
471603
prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(true)).setSettings(settings).get();
472604
Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
@@ -555,11 +687,118 @@ public void testCompositeIndexWithArraysInCompositeField() throws IOException {
555687
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
556688
);
557689
assertEquals(
558-
"object mapping for [_doc] with array for [numeric_dv] cannot be accepted as field is also part of composite index mapping which does not accept arrays",
690+
"object mapping for [_doc] with array for [numeric_dv] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
559691
ex.getMessage()
560692
);
561693
}
562694

695+
public void testCompositeIndexWithArraysInNestedCompositeField() throws IOException {
696+
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
697+
// Attempt to index a document with an array field
698+
XContentBuilder doc = jsonBuilder().startObject()
699+
.field("timestamp", "2023-06-01T12:00:00Z")
700+
.startArray("nested")
701+
.startObject()
702+
.startArray("nested1")
703+
.startObject()
704+
.field("status", 10)
705+
.endObject()
706+
.startObject()
707+
.field("status", 10)
708+
.endObject()
709+
.startObject()
710+
.field("status", 10)
711+
.endObject()
712+
.endArray()
713+
.endObject()
714+
.endArray()
715+
.endObject();
716+
// Index the document and refresh
717+
MapperParsingException ex = expectThrows(
718+
MapperParsingException.class,
719+
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
720+
);
721+
assertEquals(
722+
"object mapping for [_doc] with array for [nested] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
723+
ex.getMessage()
724+
);
725+
}
726+
727+
public void testCompositeIndexWithArraysInChildNestedCompositeField() throws IOException {
728+
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
729+
// Attempt to index a document with an array field
730+
XContentBuilder doc = jsonBuilder().startObject()
731+
.field("timestamp", "2023-06-01T12:00:00Z")
732+
.startObject("nested")
733+
.startArray("nested1")
734+
.startObject()
735+
.field("status", 10)
736+
.endObject()
737+
.startObject()
738+
.field("status", 10)
739+
.endObject()
740+
.startObject()
741+
.field("status", 10)
742+
.endObject()
743+
.endArray()
744+
.endObject()
745+
.endObject();
746+
// Index the document and refresh
747+
MapperParsingException ex = expectThrows(
748+
MapperParsingException.class,
749+
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
750+
);
751+
assertEquals(
752+
"object mapping for [nested] with array for [nested1] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
753+
ex.getMessage()
754+
);
755+
}
756+
757+
public void testCompositeIndexWithNestedArraysInNonCompositeField() throws IOException {
758+
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
759+
// Attempt to index a document with an array field
760+
XContentBuilder doc = jsonBuilder().startObject()
761+
.field("timestamp", "2023-06-01T12:00:00Z")
762+
.startObject("nested-not-startree")
763+
.startArray("nested1")
764+
.startObject()
765+
.field("status", 10)
766+
.endObject()
767+
.startObject()
768+
.field("status", 20)
769+
.endObject()
770+
.startObject()
771+
.field("status", 30)
772+
.endObject()
773+
.endArray()
774+
.endObject()
775+
.endObject();
776+
777+
// Index the document and refresh
778+
IndexResponse indexResponse = client().prepareIndex(TEST_INDEX).setSource(doc).get();
779+
780+
assertEquals(RestStatus.CREATED, indexResponse.status());
781+
782+
client().admin().indices().prepareRefresh(TEST_INDEX).get();
783+
// Verify the document was indexed
784+
SearchResponse searchResponse = client().prepareSearch(TEST_INDEX).setQuery(QueryBuilders.matchAllQuery()).get();
785+
786+
assertEquals(1, searchResponse.getHits().getTotalHits().value);
787+
788+
// Verify the values in the indexed document
789+
SearchHit hit = searchResponse.getHits().getAt(0);
790+
assertEquals("2023-06-01T12:00:00Z", hit.getSourceAsMap().get("timestamp"));
791+
792+
List<Object> values = (List<Object>) ((Map<String, Object>) (hit.getSourceAsMap().get("nested-not-startree"))).get("nested1");
793+
assertEquals(3, values.size());
794+
int i = 1;
795+
for (Object val : values) {
796+
Map<String, Object> valMap = (Map<String, Object>) val;
797+
assertEquals(10 * i, valMap.get("status"));
798+
i++;
799+
}
800+
}
801+
563802
public void testCompositeIndexWithArraysInNonCompositeField() throws IOException {
564803
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get();
565804
// Attempt to index a document with an array field

‎server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -662,11 +662,13 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp
662662
XContentParser parser = context.parser();
663663
XContentParser.Token token;
664664
// block array values for composite index fields
665-
if (context.indexSettings().isCompositeIndex() && context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)) {
665+
if (context.indexSettings().isCompositeIndex()
666+
&& (context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)
667+
|| context.mapperService().isCompositeIndexFieldNestedField(context.path().pathAsText(arrayFieldName)))) {
666668
throw new MapperParsingException(
667669
String.format(
668670
Locale.ROOT,
669-
"object mapping for [%s] with array for [%s] cannot be accepted as field is also part of composite index mapping which does not accept arrays",
671+
"object mapping for [%s] with array for [%s] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
670672
mapper.name(),
671673
arrayFieldName
672674
)

‎server/src/main/java/org/opensearch/index/mapper/MapperService.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ public enum MergeReason {
228228

229229
private volatile Set<CompositeMappedFieldType> compositeMappedFieldTypes;
230230
private volatile Set<String> fieldsPartOfCompositeMappings;
231+
private volatile Set<String> nestedFieldsPartOfCompositeMappings;
231232

232233
public MapperService(
233234
IndexSettings indexSettings,
@@ -554,10 +555,29 @@ private synchronized Map<String, DocumentMapper> internalMerge(DocumentMapper ma
554555

555556
private void buildCompositeFieldLookup() {
556557
Set<String> fieldsPartOfCompositeMappings = new HashSet<>();
558+
Set<String> nestedFieldsPartOfCompositeMappings = new HashSet<>();
559+
557560
for (CompositeMappedFieldType fieldType : compositeMappedFieldTypes) {
558561
fieldsPartOfCompositeMappings.addAll(fieldType.fields());
562+
563+
for (String field : fieldType.fields()) {
564+
String[] parts = field.split("\\.");
565+
if (parts.length > 1) {
566+
StringBuilder path = new StringBuilder();
567+
for (int i = 0; i < parts.length; i++) {
568+
if (i == 0) {
569+
path.append(parts[i]);
570+
} else {
571+
path.append(".").append(parts[i]);
572+
}
573+
nestedFieldsPartOfCompositeMappings.add(path.toString());
574+
}
575+
}
576+
}
559577
}
578+
560579
this.fieldsPartOfCompositeMappings = fieldsPartOfCompositeMappings;
580+
this.nestedFieldsPartOfCompositeMappings = nestedFieldsPartOfCompositeMappings;
561581
}
562582

563583
private boolean assertSerialization(DocumentMapper mapper) {
@@ -690,6 +710,10 @@ public boolean isFieldPartOfCompositeIndex(String field) {
690710
return fieldsPartOfCompositeMappings.contains(field);
691711
}
692712

713+
public boolean isCompositeIndexFieldNestedField(String field) {
714+
return nestedFieldsPartOfCompositeMappings.contains(field);
715+
}
716+
693717
public ObjectMapper getObjectMapper(String name) {
694718
return this.mapper == null ? null : this.mapper.objectMappers().get(name);
695719
}

0 commit comments

Comments
 (0)