Skip to content

Commit cd0e4fd

Browse files
authored
fix: correct handling of gauge metrics in renderChartConfig (#654)
1 parent a8a1f81 commit cd0e4fd

File tree

6 files changed

+287
-25
lines changed

6 files changed

+287
-25
lines changed

.changeset/tasty-bats-refuse.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
"@hyperdx/api": patch
4+
"@hyperdx/app": patch
5+
---
6+
7+
fix: correct handling of gauge metrics in renderChartConfig

packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,83 @@ Array [
3434
]
3535
`;
3636

37+
exports[`renderChartConfig Query Metrics single avg gauge with group-by 1`] = `
38+
Array [
39+
Object {
40+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
41+
"arrayElement(ResourceAttributes, 'host')": "host2",
42+
"avg(toFloat64OrNull(toString(LastValue)))": 4,
43+
},
44+
Object {
45+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
46+
"arrayElement(ResourceAttributes, 'host')": "host1",
47+
"avg(toFloat64OrNull(toString(LastValue)))": 6.25,
48+
},
49+
Object {
50+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
51+
"arrayElement(ResourceAttributes, 'host')": "host2",
52+
"avg(toFloat64OrNull(toString(LastValue)))": 4,
53+
},
54+
Object {
55+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
56+
"arrayElement(ResourceAttributes, 'host')": "host1",
57+
"avg(toFloat64OrNull(toString(LastValue)))": 80,
58+
},
59+
]
60+
`;
61+
62+
exports[`renderChartConfig Query Metrics single avg gauge with where 1`] = `
63+
Array [
64+
Object {
65+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
66+
"avg(toFloat64OrNull(toString(LastValue)))": 6.25,
67+
},
68+
Object {
69+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
70+
"avg(toFloat64OrNull(toString(LastValue)))": 80,
71+
},
72+
]
73+
`;
74+
75+
exports[`renderChartConfig Query Metrics single max/avg/sum gauge 1`] = `
76+
Array [
77+
Object {
78+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
79+
"avg(toFloat64OrNull(toString(LastValue)))": 5.125,
80+
},
81+
Object {
82+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
83+
"avg(toFloat64OrNull(toString(LastValue)))": 42,
84+
},
85+
]
86+
`;
87+
88+
exports[`renderChartConfig Query Metrics single max/avg/sum gauge 2`] = `
89+
Array [
90+
Object {
91+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
92+
"max(toFloat64OrNull(toString(LastValue)))": 6.25,
93+
},
94+
Object {
95+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
96+
"max(toFloat64OrNull(toString(LastValue)))": 80,
97+
},
98+
]
99+
`;
100+
101+
exports[`renderChartConfig Query Metrics single max/avg/sum gauge 3`] = `
102+
Array [
103+
Object {
104+
"__hdx_time_bucket": "2022-01-05T00:00:00Z",
105+
"sum(toFloat64OrNull(toString(LastValue)))": 10.25,
106+
},
107+
Object {
108+
"__hdx_time_bucket": "2022-01-05T00:05:00Z",
109+
"sum(toFloat64OrNull(toString(LastValue)))": 84,
110+
},
111+
]
112+
`;
113+
37114
exports[`renderChartConfig Query Metrics single sum rate 1`] = `
38115
Array [
39116
Object {

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

Lines changed: 108 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -340,14 +340,114 @@ describe('renderChartConfig', () => {
340340
]);
341341
});
342342

343-
it.skip('gauge (last value)', async () => {
344-
// IMPLEMENT ME (last_value aggregation)
343+
it('single max/avg/sum gauge', async () => {
344+
const avgQuery = await renderChartConfig(
345+
{
346+
select: [
347+
{
348+
aggFn: 'avg',
349+
metricName: 'test.cpu',
350+
metricType: MetricsDataType.Gauge,
351+
valueExpression: 'Value',
352+
},
353+
],
354+
from: metricSource.from,
355+
where: '',
356+
metricTables: {
357+
sum: DEFAULT_METRICS_TABLE.SUM,
358+
gauge: DEFAULT_METRICS_TABLE.GAUGE,
359+
histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
360+
},
361+
dateRange: [new Date(now), new Date(now + ms('10m'))],
362+
granularity: '5 minute',
363+
timestampValueExpression: metricSource.timestampValueExpression,
364+
connection: connection.id,
365+
},
366+
metadata,
367+
);
368+
expect(await queryData(avgQuery)).toMatchSnapshot();
369+
const maxQuery = await renderChartConfig(
370+
{
371+
select: [
372+
{
373+
aggFn: 'max',
374+
metricName: 'test.cpu',
375+
metricType: MetricsDataType.Gauge,
376+
valueExpression: 'Value',
377+
},
378+
],
379+
from: metricSource.from,
380+
where: '',
381+
metricTables: {
382+
sum: DEFAULT_METRICS_TABLE.SUM,
383+
gauge: DEFAULT_METRICS_TABLE.GAUGE,
384+
histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
385+
},
386+
dateRange: [new Date(now), new Date(now + ms('10m'))],
387+
granularity: '5 minute',
388+
timestampValueExpression: metricSource.timestampValueExpression,
389+
connection: connection.id,
390+
},
391+
metadata,
392+
);
393+
expect(await queryData(maxQuery)).toMatchSnapshot();
394+
const sumQuery = await renderChartConfig(
395+
{
396+
select: [
397+
{
398+
aggFn: 'sum',
399+
metricName: 'test.cpu',
400+
metricType: MetricsDataType.Gauge,
401+
valueExpression: 'Value',
402+
},
403+
],
404+
from: metricSource.from,
405+
where: '',
406+
metricTables: {
407+
sum: DEFAULT_METRICS_TABLE.SUM,
408+
gauge: DEFAULT_METRICS_TABLE.GAUGE,
409+
histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
410+
},
411+
dateRange: [new Date(now), new Date(now + ms('10m'))],
412+
granularity: '5 minute',
413+
timestampValueExpression: metricSource.timestampValueExpression,
414+
connection: connection.id,
415+
},
416+
metadata,
417+
);
418+
expect(await queryData(sumQuery)).toMatchSnapshot();
419+
});
420+
421+
it('single avg gauge with where', async () => {
422+
const query = await renderChartConfig(
423+
{
424+
select: [
425+
{
426+
aggFn: 'avg',
427+
metricName: 'test.cpu',
428+
metricType: MetricsDataType.Gauge,
429+
valueExpression: 'Value',
430+
},
431+
],
432+
from: metricSource.from,
433+
where: `ResourceAttributes['host'] = 'host1'`,
434+
whereLanguage: 'sql',
435+
metricTables: {
436+
sum: DEFAULT_METRICS_TABLE.SUM,
437+
gauge: DEFAULT_METRICS_TABLE.GAUGE,
438+
histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
439+
},
440+
dateRange: [new Date(now), new Date(now + ms('10m'))],
441+
granularity: '5 minute',
442+
timestampValueExpression: metricSource.timestampValueExpression,
443+
connection: connection.id,
444+
},
445+
metadata,
446+
);
447+
expect(await queryData(query)).toMatchSnapshot();
345448
});
346449

347-
// FIXME: gauge avg doesn't work as expected (it should pull the average of the last value)
348-
// in this case
349-
// (6.25 + 4) / 2, (80 + 4) / 2
350-
it('single avg gauge', async () => {
450+
it('single avg gauge with group-by', async () => {
351451
const query = await renderChartConfig(
352452
{
353453
select: [
@@ -367,11 +467,13 @@ describe('renderChartConfig', () => {
367467
},
368468
dateRange: [new Date(now), new Date(now + ms('10m'))],
369469
granularity: '5 minute',
470+
groupBy: `ResourceAttributes['host']`,
370471
timestampValueExpression: metricSource.timestampValueExpression,
371472
connection: connection.id,
372473
},
373474
metadata,
374475
);
476+
expect(await queryData(query)).toMatchSnapshot();
375477
});
376478

377479
it('single sum rate', async () => {
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
4+
"WITH Bucketed AS (
5+
SELECT
6+
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`,
7+
ScopeAttributes,
8+
ResourceAttributes,
9+
Attributes,
10+
cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
11+
last_value(Value) AS LastValue,
12+
any(ResourceSchemaUrl) AS ResourceSchemaUrl,
13+
any(ScopeName) AS ScopeName,
14+
any(ScopeVersion) AS ScopeVersion,
15+
any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
16+
any(ScopeSchemaUrl) AS ScopeSchemaUrl,
17+
any(ServiceName) AS ServiceName,
18+
any(MetricDescription) AS MetricDescription,
19+
any(MetricUnit) AS MetricUnit,
20+
any(StartTimeUnix) AS StartTimeUnix,
21+
any(Flags) AS Flags
22+
FROM default.otel_metrics_gauge
23+
WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'nodejs.event_loop.utilization'))
24+
GROUP BY ScopeAttributes, ResourceAttributes, Attributes, __hdx_time_bucket2
25+
ORDER BY AttributesHash, __hdx_time_bucket2
26+
) SELECT quantile(0.95)(toFloat64OrNull(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))
27+
TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))
28+
STEP 60 LIMIT 10"
29+
`;

packages/common-utils/src/__tests__/renderChartConfig.test.ts

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ describe('renderChartConfig', () => {
1717
{ name: 'timestamp', type: 'DateTime' },
1818
{ name: 'value', type: 'Float64' },
1919
]),
20-
getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue({}),
20+
getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
2121
getColumn: jest.fn().mockResolvedValue({ type: 'DateTime' }),
2222
} as unknown as Metadata;
2323
});
@@ -57,16 +57,7 @@ describe('renderChartConfig', () => {
5757

5858
const generatedSql = await renderChartConfig(config, mockMetadata);
5959
const actual = parameterizedQueryToSql(generatedSql);
60-
expect(actual).toBe(
61-
'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
62-
' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' +
63-
" (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" +
64-
' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
65-
' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' +
66-
' TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' +
67-
' STEP 60' +
68-
' LIMIT 10',
69-
);
60+
expect(actual).toMatchSnapshot();
7061
});
7162

7263
it('should generate sql for a single sum metric', async () => {

packages/common-utils/src/renderChartConfig.ts

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ function determineTableName(select: SelectSQLStatement): string {
3838
return '';
3939
}
4040

41+
const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix';
4142
export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';
4243

4344
export function isUsingGroupBy(
@@ -784,9 +785,10 @@ function renderFill(
784785
return undefined;
785786
}
786787

787-
function translateMetricChartConfig(
788+
async function translateMetricChartConfig(
788789
chartConfig: ChartConfigWithOptDateRange,
789-
): ChartConfigWithOptDateRangeEx {
790+
metadata: Metadata,
791+
): Promise<ChartConfigWithOptDateRangeEx> {
790792
const metricTables = chartConfig.metricTables;
791793
if (!metricTables) {
792794
return chartConfig;
@@ -800,20 +802,74 @@ function translateMetricChartConfig(
800802

801803
const { metricType, metricName, ..._select } = select[0]; // Initial impl only supports one metric select per chart config
802804
if (metricType === MetricsDataType.Gauge && metricName) {
805+
const timeBucketCol = '__hdx_time_bucket2';
806+
const timeExpr = timeBucketExpr({
807+
interval: chartConfig.granularity || 'auto',
808+
timestampValueExpression:
809+
chartConfig.timestampValueExpression ||
810+
DEFAULT_METRIC_TABLE_TIME_COLUMN,
811+
dateRange: chartConfig.dateRange,
812+
alias: timeBucketCol,
813+
});
814+
815+
const where = await renderWhere(
816+
{
817+
...chartConfig,
818+
from: {
819+
...from,
820+
tableName: metricTables[MetricsDataType.Gauge],
821+
},
822+
filters: [
823+
{
824+
type: 'sql',
825+
condition: `MetricName = '${metricName}'`,
826+
},
827+
],
828+
},
829+
metadata,
830+
);
831+
803832
return {
804833
...restChartConfig,
834+
with: [
835+
{
836+
name: 'Bucketed',
837+
sql: chSql`
838+
SELECT
839+
${timeExpr},
840+
ScopeAttributes,
841+
ResourceAttributes,
842+
Attributes,
843+
cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
844+
last_value(Value) AS LastValue,
845+
any(ResourceSchemaUrl) AS ResourceSchemaUrl,
846+
any(ScopeName) AS ScopeName,
847+
any(ScopeVersion) AS ScopeVersion,
848+
any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
849+
any(ScopeSchemaUrl) AS ScopeSchemaUrl,
850+
any(ServiceName) AS ServiceName,
851+
any(MetricDescription) AS MetricDescription,
852+
any(MetricUnit) AS MetricUnit,
853+
any(StartTimeUnix) AS StartTimeUnix,
854+
any(Flags) AS Flags
855+
FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })}
856+
WHERE ${where}
857+
GROUP BY ScopeAttributes, ResourceAttributes, Attributes, ${timeBucketCol}
858+
ORDER BY AttributesHash, ${timeBucketCol}
859+
`,
860+
},
861+
],
805862
select: [
806863
{
807864
..._select,
808-
valueExpression: 'Value',
865+
valueExpression: 'LastValue',
809866
},
810867
],
811868
from: {
812-
...from,
813-
tableName: metricTables[MetricsDataType.Gauge],
869+
databaseName: '',
870+
tableName: 'Bucketed',
814871
},
815-
where: `MetricName = '${metricName}'`,
816-
whereLanguage: 'sql',
872+
timestampValueExpression: timeBucketCol,
817873
};
818874
} else if (metricType === MetricsDataType.Sum && metricName) {
819875
return {
@@ -921,7 +977,7 @@ export async function renderChartConfig(
921977
// but goes through the same generation process
922978
const chartConfig =
923979
rawChartConfig.metricTables != null
924-
? translateMetricChartConfig(rawChartConfig)
980+
? await translateMetricChartConfig(rawChartConfig, metadata)
925981
: rawChartConfig;
926982

927983
const withClauses = renderWith(chartConfig, metadata);

0 commit comments

Comments
 (0)