Skip to content

Commit c20f12c

Browse files
bharath-techiebharath-techie
committed
Merge branch 'main' of github.com:opensearch-project/OpenSearch into count_avg_mapper
2 parents 0777a28 + e8ee6db commit c20f12c

20 files changed

+151
-80
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ public void testValidCompositeIndex() {
267267
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
268268

269269
// Assert default metrics
270-
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM, MetricStat.AVG);
270+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
271271
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
272272
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
273273
assertEquals(
@@ -347,7 +347,7 @@ public void testUpdateIndexWhenMappingIsSame() {
347347
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
348348

349349
// Assert default metrics
350-
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM, MetricStat.AVG);
350+
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
351351
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
352352
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
353353
assertEquals(

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,17 @@
2020
*/
2121
@ExperimentalApi
2222
public enum MetricStat {
23-
COUNT("count"),
23+
VALUE_COUNT("value_count"),
2424
SUM("sum"),
2525
MIN("min"),
2626
MAX("max"),
27-
AVG("avg", COUNT, SUM);
27+
AVG("avg", VALUE_COUNT, SUM);
2828

2929
private final String typeName;
3030
private final MetricStat[] baseMetrics;
3131

3232
MetricStat(String typeName, MetricStat... baseMetrics) {
3333
this.typeName = typeName;
34-
assert baseMetrics != null && baseMetrics.length > 0;
3534
this.baseMetrics = baseMetrics;
3635
}
3736

@@ -52,7 +51,7 @@ public List<MetricStat> getBaseMetrics() {
5251
* For example, AVG is derived from COUNT and SUM
5352
*/
5453
public boolean isDerivedMetric() {
55-
return baseMetrics != null;
54+
return baseMetrics != null && baseMetrics.length > 0;
5655
}
5756

5857
public static MetricStat fromTypeName(String typeName) {

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public class StarTreeIndexSettings {
9696
*/
9797
public static final Setting<List<String>> DEFAULT_METRICS_LIST = Setting.listSetting(
9898
"index.composite_index.star_tree.field.default.metrics",
99-
Arrays.asList(MetricStat.COUNT.toString(), MetricStat.SUM.toString()),
99+
Arrays.asList(MetricStat.VALUE_COUNT.toString(), MetricStat.SUM.toString()),
100100
Function.identity(),
101101
Setting.Property.IndexScope,
102102
Setting.Property.Final

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public Long mergeAggregatedValues(Long value, Long aggregatedValue) {
6060
}
6161

6262
@Override
63-
public Long toStarTreeNumericTypeValue(Long value) {
63+
public Long toAggregatedValueType(Long value) {
6464
return value;
6565
}
6666

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,12 @@ public Double mergeAggregatedValues(Double value, Double aggregatedValue) {
5252
}
5353

5454
@Override
55-
public Double toStarTreeNumericTypeValue(Long value) {
55+
public Double toAggregatedValueType(Long value) {
5656
try {
5757
if (value == null) {
5858
return getIdentityMetricValue();
5959
}
60-
return starTreeNumericType.getDoubleValue(value);
60+
return VALUE_AGGREGATOR_TYPE.getDoubleValue(value);
6161
} catch (Exception e) {
6262
throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e);
6363
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,12 @@ public Double getInitialAggregatedValue(Double value) {
8787
}
8888

8989
@Override
90-
public Double toStarTreeNumericTypeValue(Long value) {
90+
public Double toAggregatedValueType(Long value) {
9191
try {
9292
if (value == null) {
9393
return getIdentityMetricValue();
9494
}
95-
return starTreeNumericType.getDoubleValue(value);
95+
return VALUE_AGGREGATOR_TYPE.getDoubleValue(value);
9696
} catch (Exception e) {
9797
throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e);
9898
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ default A getInitialAggregatedValue(A value) {
5050
}
5151

5252
/**
53-
* Converts an aggregated value from a Long type.
53+
* Converts a segment long value to an aggregated value.
5454
*/
55-
A toStarTreeNumericTypeValue(Long rawValue);
55+
A toAggregatedValueType(Long rawValue);
5656

5757
/**
5858
* Fetches a value that does not alter the result of aggregations

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType, Sta
3030
// avg aggregator will be covered in the part of query (using count and sum)
3131
case SUM:
3232
return new SumValueAggregator(starTreeNumericType);
33-
case COUNT:
33+
case VALUE_COUNT:
3434
return new CountValueAggregator(starTreeNumericType);
3535
case MIN:
3636
return new MinValueAggregator(starTreeNumericType);

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ protected StarTreeDocument getStarTreeDocument(
167167
// actual indexing field they're based on
168168
metrics[i] = metricAggregatorInfos.get(i)
169169
.getValueAggregators()
170-
.toStarTreeNumericTypeValue(metricDocValuesIterator.value(currentDocId));
170+
.toAggregatedValueType(metricDocValuesIterator.value(currentDocId));
171171
i++;
172172
}
173173
return new StarTreeDocument(dims, metrics);

‎server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.common.util;
10+
11+
import org.apache.lucene.store.FSDirectory;
12+
import org.apache.lucene.store.IOContext;
13+
import org.apache.lucene.store.IndexInput;
14+
import org.apache.lucene.store.IndexOutput;
15+
import org.apache.lucene.store.RandomAccessInput;
16+
import org.opensearch.test.OpenSearchTestCase;
17+
18+
import java.io.IOException;
19+
import java.nio.file.Path;
20+
21+
/**
22+
* Tests for {@link ByteArrayBackedBitset}
23+
*/
24+
public class ByteArrayBackedBitsetTests extends OpenSearchTestCase {
25+
public void testWriteAndReadNullBitSets() throws IOException {
26+
for (int k = 0; k < 10; k++) {
27+
int randomArraySize = randomIntBetween(2, 300);
28+
int randomIndex1 = randomIntBetween(0, (randomArraySize - 1) * 8);
29+
int randomIndex2 = randomIntBetween(0, (randomArraySize - 1) * 8);
30+
testWriteAndReadBitset(randomArraySize, randomIndex1, randomIndex2);
31+
}
32+
}
33+
34+
private static void testWriteAndReadBitset(int randomArraySize, int randomIndex1, int randomIndex2) throws IOException {
35+
ByteArrayBackedBitset bitset = new ByteArrayBackedBitset(randomArraySize);
36+
Path basePath = createTempDir("OffHeapTests");
37+
FSDirectory fsDirectory = FSDirectory.open(basePath);
38+
String TEST_FILE = "test_file";
39+
IndexOutput indexOutput = fsDirectory.createOutput(TEST_FILE, IOContext.DEFAULT);
40+
bitset.set(randomIndex1);
41+
bitset.set(randomIndex2);
42+
bitset.write(indexOutput);
43+
indexOutput.close();
44+
45+
IndexInput in = fsDirectory.openInput(TEST_FILE, IOContext.DEFAULT);
46+
RandomAccessInput randomAccessInput = in.randomAccessSlice(0, in.length());
47+
ByteArrayBackedBitset bitset1 = new ByteArrayBackedBitset(randomAccessInput, 0, randomArraySize);
48+
for (int i = 0; i < (randomArraySize * 8); i++) {
49+
if (randomIndex1 == i || randomIndex2 == i) {
50+
assertTrue(bitset1.get(i));
51+
} else {
52+
assertFalse(bitset1.get(i));
53+
}
54+
}
55+
}
56+
}

‎server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE
126126
b.field("name", "field");
127127
b.startArray("stats");
128128
b.value("sum");
129-
b.value("count"); // TODO : THIS TEST FAILS.
129+
b.value("value_count");
130130
b.endArray();
131131
b.endObject();
132132
b.endArray();

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
public abstract class AbstractValueAggregatorTests extends OpenSearchTestCase {
2222

2323
private ValueAggregator aggregator;
24-
private StarTreeNumericType starTreeNumericType;
24+
protected StarTreeNumericType starTreeNumericType;
2525

2626
public AbstractValueAggregatorTests(StarTreeNumericType starTreeNumericType) {
2727
this.starTreeNumericType = starTreeNumericType;
@@ -69,7 +69,7 @@ public void testGetInitialAggregatedValueForSegmentDocValue() {
6969
assertEquals(CountValueAggregator.DEFAULT_INITIAL_VALUE, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong()));
7070
} else {
7171
assertEquals(
72-
aggregator.toStarTreeNumericTypeValue(randomLong),
72+
starTreeNumericType.getDoubleValue(randomLong),
7373
aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong)
7474
);
7575
}

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ public void testGetInitialAggregatedValue() {
3636
assertEquals(randomLong, aggregator.getInitialAggregatedValue(randomLong), 0.0);
3737
}
3838

39-
public void testToStarTreeNumericTypeValue() {
39+
public void testToAggregatedValueType() {
4040
long randomLong = randomLong();
41-
assertEquals(randomLong, aggregator.toStarTreeNumericTypeValue(randomLong), 0.0);
42-
assertNull(aggregator.toStarTreeNumericTypeValue(null));
41+
assertEquals(randomLong, aggregator.toAggregatedValueType(randomLong), 0.0);
42+
assertNull(aggregator.toAggregatedValueType(null));
4343
}
4444

4545
public void testIdentityMetricValue() {

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java

+5-13
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,13 @@ public void testMergeAggregatedValueAndSegmentValue() {
2323
Long randomLong = randomLong();
2424
double randomDouble = randomDouble();
2525
assertEquals(
26-
Math.max(aggregator.toStarTreeNumericTypeValue(randomLong), randomDouble),
26+
Math.max(starTreeNumericType.getDoubleValue(randomLong), randomDouble),
2727
aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong),
2828
0.0
2929
);
30-
assertEquals(
31-
aggregator.toStarTreeNumericTypeValue(randomLong),
32-
aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong),
33-
0.0
34-
);
30+
assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0);
3531
assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0);
36-
assertEquals(
37-
Math.max(2.0, aggregator.toStarTreeNumericTypeValue(3L)),
38-
aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L),
39-
0.0
40-
);
32+
assertEquals(Math.max(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0);
4133
}
4234

4335
public void testMergeAggregatedValues() {
@@ -53,10 +45,10 @@ public void testGetInitialAggregatedValue() {
5345
assertEquals(randomDouble, aggregator.getInitialAggregatedValue(randomDouble), 0.0);
5446
}
5547

56-
public void testToStarTreeNumericTypeValue() {
48+
public void testToAggregatedValueType() {
5749
MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE);
5850
long randomLong = randomLong();
59-
assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toStarTreeNumericTypeValue(randomLong), 0.0);
51+
assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0);
6052
}
6153

6254
public void testIdentityMetricValue() {

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ public void testConstructor() {
2727

2828
public void testCountStarConstructor() {
2929
MetricAggregatorInfo pair = new MetricAggregatorInfo(
30-
MetricStat.COUNT,
30+
MetricStat.VALUE_COUNT,
3131
"anything",
3232
"star_tree_field",
3333
IndexNumericFieldData.NumericType.DOUBLE
3434
);
35-
assertEquals(MetricStat.COUNT, pair.getMetricStat());
35+
assertEquals(MetricStat.VALUE_COUNT, pair.getMetricStat());
3636
assertEquals("anything", pair.getField());
3737
}
3838

@@ -62,7 +62,7 @@ public void testEquals() {
6262
assertEquals(pair1, pair2);
6363
assertNotEquals(
6464
pair1,
65-
new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE)
65+
new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE)
6666
);
6767
assertNotEquals(
6868
pair1,
@@ -100,7 +100,7 @@ public void testCompareTo() {
100100
IndexNumericFieldData.NumericType.DOUBLE
101101
);
102102
MetricAggregatorInfo pair3 = new MetricAggregatorInfo(
103-
MetricStat.COUNT,
103+
MetricStat.VALUE_COUNT,
104104
"column1",
105105
"star_tree_field",
106106
IndexNumericFieldData.NumericType.DOUBLE

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java

+5-13
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,13 @@ public void testMergeAggregatedValueAndSegmentValue() {
2222
Long randomLong = randomLong();
2323
double randomDouble = randomDouble();
2424
assertEquals(
25-
Math.min(aggregator.toStarTreeNumericTypeValue(randomLong), randomDouble),
25+
Math.min(starTreeNumericType.getDoubleValue(randomLong), randomDouble),
2626
aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong),
2727
0.0
2828
);
29-
assertEquals(
30-
aggregator.toStarTreeNumericTypeValue(randomLong),
31-
aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong),
32-
0.0
33-
);
29+
assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0);
3430
assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0);
35-
assertEquals(
36-
Math.min(2.0, aggregator.toStarTreeNumericTypeValue(3L)),
37-
aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L),
38-
0.0
39-
);
31+
assertEquals(Math.min(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0);
4032
}
4133

4234
public void testMergeAggregatedValues() {
@@ -52,10 +44,10 @@ public void testGetInitialAggregatedValue() {
5244
assertEquals(randomDouble, aggregator.getInitialAggregatedValue(randomDouble), 0.0);
5345
}
5446

55-
public void testToStarTreeNumericTypeValue() {
47+
public void testToAggregatedValueType() {
5648
MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE);
5749
long randomLong = randomLong();
58-
assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toStarTreeNumericTypeValue(randomLong), 0.0);
50+
assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0);
5951
}
6052

6153
public void testIdentityMetricValue() {

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java

+5-9
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public void testMergeAggregatedValueAndSegmentValue() {
2929
Long randomLong = randomLong();
3030
aggregator.getInitialAggregatedValue(randomDouble);
3131
assertEquals(
32-
randomDouble + aggregator.toStarTreeNumericTypeValue(randomLong),
32+
randomDouble + starTreeNumericType.getDoubleValue(randomLong),
3333
aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong),
3434
0.0
3535
);
@@ -41,7 +41,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() {
4141
aggregator.getInitialAggregatedValue(randomDouble1);
4242
assertEquals(randomDouble1, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, null), 0.0);
4343
assertEquals(
44-
randomDouble1 + aggregator.toStarTreeNumericTypeValue(randomLong),
44+
randomDouble1 + starTreeNumericType.getDoubleValue(randomLong),
4545
aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, randomLong),
4646
0.0
4747
);
@@ -50,11 +50,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() {
5050
public void testMergeAggregatedValueAndSegmentValue_nullInitialDocValue() {
5151
Long randomLong = randomLong();
5252
aggregator.getInitialAggregatedValue(null);
53-
assertEquals(
54-
aggregator.toStarTreeNumericTypeValue(randomLong),
55-
aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong),
56-
0.0
57-
);
53+
assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0);
5854
}
5955

6056
public void testMergeAggregatedValues() {
@@ -70,9 +66,9 @@ public void testGetInitialAggregatedValue() {
7066
assertEquals(randomDouble, aggregator.getInitialAggregatedValue(randomDouble), 0.0);
7167
}
7268

73-
public void testToStarTreeNumericTypeValue() {
69+
public void testToAggregatedValueType() {
7470
long randomLong = randomLong();
75-
assertEquals(aggregator.toStarTreeNumericTypeValue(randomLong), aggregator.toStarTreeNumericTypeValue(randomLong), 0.0);
71+
assertEquals(aggregator.toAggregatedValueType(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0);
7672
}
7773

7874
public void testIdentityMetricValue() {

‎server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public void testGetValueAggregatorForMaxType() {
3333
}
3434

3535
public void testGetValueAggregatorForCountType() {
36-
ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.COUNT, StarTreeNumericType.LONG);
36+
ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.VALUE_COUNT, StarTreeNumericType.LONG);
3737
assertNotNull(aggregator);
3838
assertEquals(CountValueAggregator.class, aggregator.getClass());
3939
}

0 commit comments

Comments
 (0)