Skip to content

Commit e09106d

Browse files
bharath-techiebharath-techie
committed
Changes to handle count and avg metrics as part of star tree mapping
Signed-off-by: Bharathwaj G <[email protected]>
1 parent a918530 commit e09106d

File tree

5 files changed

+148
-49
lines changed

5 files changed

+148
-49
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+6-14
Original file line numberDiff line numberDiff line change
@@ -265,13 +265,9 @@ public void testValidCompositeIndex() {
265265
assertEquals(expectedTimeUnits, dateDim.getIntervals());
266266
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
267267
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
268-
List<MetricStat> expectedMetrics = Arrays.asList(
269-
MetricStat.AVG,
270-
MetricStat.COUNT,
271-
MetricStat.SUM,
272-
MetricStat.MAX,
273-
MetricStat.MIN
274-
);
268+
269+
// Assert default metrics
270+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
275271
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
276272
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
277273
assertEquals(
@@ -349,13 +345,9 @@ public void testUpdateIndexWhenMappingIsSame() {
349345
assertEquals(expectedTimeUnits, dateDim.getIntervals());
350346
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
351347
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
352-
List<MetricStat> expectedMetrics = Arrays.asList(
353-
MetricStat.AVG,
354-
MetricStat.COUNT,
355-
MetricStat.SUM,
356-
MetricStat.MAX,
357-
MetricStat.MIN
358-
);
348+
349+
// Assert default metrics
350+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
359351
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
360352
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
361353
assertEquals(

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java

+36-6
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,59 @@
1010

1111
import org.opensearch.common.annotation.ExperimentalApi;
1212

13+
import java.util.Arrays;
14+
import java.util.Collections;
15+
import java.util.List;
16+
import java.util.Set;
17+
1318
/**
1419
* Supported metric types for composite index
1520
*
1621
* @opensearch.experimental
1722
*/
1823
@ExperimentalApi
1924
public enum MetricStat {
20-
COUNT("count"),
21-
AVG("avg"),
22-
SUM("sum"),
23-
MIN("min"),
24-
MAX("max");
25+
COUNT("count", null),
26+
SUM("sum", null),
27+
MIN("min", null),
28+
MAX("max", null),
29+
AVG("avg", new MetricStat[] { COUNT, SUM });
2530

2631
private final String typeName;
32+
private final MetricStat[] derivedFrom;
2733

28-
MetricStat(String typeName) {
34+
MetricStat(String typeName, MetricStat[] derivedFrom) {
2935
this.typeName = typeName;
36+
this.derivedFrom = derivedFrom;
3037
}
3138

3239
public String getTypeName() {
3340
return typeName;
3441
}
3542

43+
/**
44+
* Return the list of metrics that this metric is derived from
45+
* For example, AVG is derived from COUNT and SUM
46+
*/
47+
public List<MetricStat> getDerivedFromMetrics() {
48+
return Arrays.asList(derivedFrom);
49+
}
50+
51+
/**
52+
* Return true if this metric is derived from other metrics
53+
* For example, AVG is derived from COUNT and SUM
54+
*/
55+
public boolean isDerivedMetric() {
56+
return derivedFrom != null;
57+
}
58+
59+
/**
60+
* Return required metrics for every metric field in star tree field
61+
*/
62+
public static Set<MetricStat> getRequiredMetrics() {
63+
return Collections.singleton(MetricStat.COUNT);
64+
}
65+
3666
public static MetricStat fromTypeName(String typeName) {
3767
for (MetricStat metric : MetricStat.values()) {
3868
if (metric.getTypeName().equalsIgnoreCase(typeName)) {

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java

+4-9
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.util.Arrays;
1717
import java.util.List;
18+
import java.util.function.Function;
1819

1920
/**
2021
* Index settings for star tree fields. The settings are final as right now
@@ -93,16 +94,10 @@ public class StarTreeIndexSettings {
9394
/**
9495
* Default metrics for metrics as part of star tree fields
9596
*/
96-
public static final Setting<List<MetricStat>> DEFAULT_METRICS_LIST = Setting.listSetting(
97+
public static final Setting<List<String>> DEFAULT_METRICS_LIST = Setting.listSetting(
9798
"index.composite_index.star_tree.field.default.metrics",
98-
Arrays.asList(
99-
MetricStat.AVG.toString(),
100-
MetricStat.COUNT.toString(),
101-
MetricStat.SUM.toString(),
102-
MetricStat.MAX.toString(),
103-
MetricStat.MIN.toString()
104-
),
105-
MetricStat::fromTypeName,
99+
Arrays.asList(MetricStat.COUNT.toString(), MetricStat.SUM.toString()),
100+
Function.identity(),
106101
Setting.Property.IndexScope,
107102
Setting.Property.Final
108103
);

‎server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java

+11-6
Original file line numberDiff line numberDiff line change
@@ -262,14 +262,19 @@ private Metric getMetric(String name, Map<String, Object> metric, Mapper.TypePar
262262
.collect(Collectors.toList());
263263
metric.remove(STATS);
264264
if (metricStrings.isEmpty()) {
265-
metricTypes = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings()));
266-
} else {
267-
Set<MetricStat> metricSet = new LinkedHashSet<>();
268-
for (String metricString : metricStrings) {
269-
metricSet.add(MetricStat.fromTypeName(metricString));
265+
metricStrings = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings()));
266+
}
267+
// Add all required field initially
268+
Set<MetricStat> metricSet = new LinkedHashSet<>(MetricStat.getRequiredMetrics());
269+
for (String metricString : metricStrings) {
270+
MetricStat metricStat = MetricStat.fromTypeName(metricString);
271+
if (metricStat.isDerivedMetric()) {
272+
metricSet.addAll(metricStat.getDerivedFromMetrics());
273+
} else {
274+
metricSet.add(metricStat);
270275
}
271-
metricTypes = new ArrayList<>(metricSet);
272276
}
277+
metricTypes = new ArrayList<>(metricSet);
273278
return new Metric(name, metricTypes);
274279
}
275280

‎server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java

+91-14
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public void teardown() {
5252
}
5353

5454
public void testValidStarTree() throws IOException {
55-
MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
55+
MapperService mapperService = createMapperService(getExpandedMappingWithJustAvg("status", "size"));
5656
Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
5757
for (CompositeMappedFieldType type : compositeFieldTypes) {
5858
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
@@ -66,7 +66,37 @@ public void testValidStarTree() throws IOException {
6666
assertEquals(expectedTimeUnits, dateDim.getIntervals());
6767
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
6868
assertEquals("size", starTreeFieldType.getMetrics().get(0).getField());
69-
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.SUM, MetricStat.AVG);
69+
70+
// Assert COUNT and SUM gets added when AVG is mentioned
71+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
72+
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
73+
assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
74+
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
75+
assertEquals(
76+
new HashSet<>(Arrays.asList("@timestamp", "status")),
77+
starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()
78+
);
79+
}
80+
}
81+
82+
public void testMetricsWithJustSum() throws IOException {
83+
MapperService mapperService = createMapperService(getExpandedMappingWithJustSum("status", "size"));
84+
Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
85+
for (CompositeMappedFieldType type : compositeFieldTypes) {
86+
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
87+
assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField());
88+
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
89+
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
90+
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
91+
Rounding.DateTimeUnit.DAY_OF_MONTH,
92+
Rounding.DateTimeUnit.MONTH_OF_YEAR
93+
);
94+
assertEquals(expectedTimeUnits, dateDim.getIntervals());
95+
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
96+
assertEquals("size", starTreeFieldType.getMetrics().get(0).getField());
97+
98+
// Assert COUNT gets added for any metric field
99+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
70100
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
71101
assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
72102
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
@@ -92,13 +122,7 @@ public void testValidStarTreeDefaults() throws IOException {
92122
assertEquals(expectedTimeUnits, dateDim.getIntervals());
93123
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
94124
assertEquals("status", starTreeFieldType.getMetrics().get(0).getField());
95-
List<MetricStat> expectedMetrics = Arrays.asList(
96-
MetricStat.AVG,
97-
MetricStat.COUNT,
98-
MetricStat.SUM,
99-
MetricStat.MAX,
100-
MetricStat.MIN
101-
);
125+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
102126
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
103127
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
104128
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
@@ -109,15 +133,15 @@ public void testValidStarTreeDefaults() throws IOException {
109133
public void testInvalidDim() {
110134
MapperParsingException ex = expectThrows(
111135
MapperParsingException.class,
112-
() -> createMapperService(getExpandedMapping("invalid", "size"))
136+
() -> createMapperService(getExpandedMappingWithJustAvg("invalid", "size"))
113137
);
114138
assertEquals("Failed to parse mapping [_doc]: unknown dimension field [invalid]", ex.getMessage());
115139
}
116140

117141
public void testInvalidMetric() {
118142
MapperParsingException ex = expectThrows(
119143
MapperParsingException.class,
120-
() -> createMapperService(getExpandedMapping("status", "invalid"))
144+
() -> createMapperService(getExpandedMappingWithJustAvg("status", "invalid"))
121145
);
122146
assertEquals("Failed to parse mapping [_doc]: unknown metric field [invalid]", ex.getMessage());
123147
}
@@ -232,6 +256,10 @@ public void testMetric() {
232256
assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min"));
233257
assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum"));
234258
assertEquals(MetricStat.AVG, MetricStat.fromTypeName("avg"));
259+
260+
assertEquals(Set.of(MetricStat.COUNT), MetricStat.getRequiredMetrics());
261+
assertEquals(List.of(MetricStat.COUNT, MetricStat.SUM), MetricStat.AVG.getDerivedFromMetrics());
262+
235263
IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> MetricStat.fromTypeName("invalid"));
236264
assertEquals("Invalid metric stat: invalid", ex.getMessage());
237265
}
@@ -310,7 +338,7 @@ public void testStarTreeField() {
310338
}
311339

312340
public void testValidations() throws IOException {
313-
MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
341+
MapperService mapperService = createMapperService(getExpandedMappingWithJustAvg("status", "size"));
314342
Settings settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build();
315343
CompositeIndexSettings enabledCompositeIndexSettings = new CompositeIndexSettings(
316344
settings,
@@ -370,7 +398,7 @@ public void testValidations() throws IOException {
370398
);
371399
}
372400

373-
private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException {
401+
private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) throws IOException {
374402
return topMapping(b -> {
375403
b.startObject("composite");
376404
b.startObject("startree");
@@ -399,7 +427,6 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE
399427
b.startObject();
400428
b.field("name", metric);
401429
b.startArray("stats");
402-
b.value("sum");
403430
b.value("avg");
404431
b.endArray();
405432
b.endObject();
@@ -421,6 +448,56 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE
421448
});
422449
}
423450

451+
private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) throws IOException {
452+
return topMapping(b -> {
453+
b.startObject("composite");
454+
b.startObject("startree");
455+
b.field("type", "star_tree");
456+
b.startObject("config");
457+
b.field("max_leaf_docs", 100);
458+
b.startArray("skip_star_node_creation_for_dimensions");
459+
{
460+
b.value("@timestamp");
461+
b.value("status");
462+
}
463+
b.endArray();
464+
b.startArray("ordered_dimensions");
465+
b.startObject();
466+
b.field("name", "@timestamp");
467+
b.startArray("calendar_intervals");
468+
b.value("day");
469+
b.value("month");
470+
b.endArray();
471+
b.endObject();
472+
b.startObject();
473+
b.field("name", dim);
474+
b.endObject();
475+
b.endArray();
476+
b.startArray("metrics");
477+
b.startObject();
478+
b.field("name", metric);
479+
b.startArray("stats");
480+
b.value("sum");
481+
b.endArray();
482+
b.endObject();
483+
b.endArray();
484+
b.endObject();
485+
b.endObject();
486+
b.endObject();
487+
b.startObject("properties");
488+
b.startObject("@timestamp");
489+
b.field("type", "date");
490+
b.endObject();
491+
b.startObject("status");
492+
b.field("type", "integer");
493+
b.endObject();
494+
b.startObject("size");
495+
b.field("type", "integer");
496+
b.endObject();
497+
b.endObject();
498+
});
499+
}
500+
424501
private XContentBuilder getMinMapping() throws IOException {
425502
return getMinMapping(false, false, false, false);
426503
}

0 commit comments

Comments
 (0)