fix: correct handling of gauge metrics in renderChartConfig

.changeset/tasty-bats-refuse.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+fix: correct handling of gauge metrics in renderChartConfig

packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -34,6 +34,83 @@ Array [
 ]
 `;
 
+exports[`renderChartConfig Query Metrics single avg gauge with group-by 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host2",
+    "avg(toFloat64OrNull(toString(LastValue)))": 4,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host1",
+    "avg(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host2",
+    "avg(toFloat64OrNull(toString(LastValue)))": 4,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "arrayElement(ResourceAttributes, 'host')": "host1",
+    "avg(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single avg gauge with where 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 1`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 5.125,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "avg(toFloat64OrNull(toString(LastValue)))": 42,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 2`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "max(toFloat64OrNull(toString(LastValue)))": 6.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "max(toFloat64OrNull(toString(LastValue)))": 80,
+  },
+]
+`;
+
+exports[`renderChartConfig Query Metrics single max/avg/sum gauge 3`] = `
+Array [
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:00:00Z",
+    "sum(toFloat64OrNull(toString(LastValue)))": 10.25,
+  },
+  Object {
+    "__hdx_time_bucket": "2022-01-05T00:05:00Z",
+    "sum(toFloat64OrNull(toString(LastValue)))": 84,
+  },
+]
+`;
+
 exports[`renderChartConfig Query Metrics single sum rate 1`] = `
 Array [
   Object {

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

@@ -340,14 +340,114 @@ describe('renderChartConfig', () => {
       ]);
     });
 
-    it.skip('gauge (last value)', async () => {
-      // IMPLEMENT ME (last_value aggregation)
+    it('single max/avg/sum gauge', async () => {
+      const avgQuery = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'avg',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(avgQuery)).toMatchSnapshot();
+      const maxQuery = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'max',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(maxQuery)).toMatchSnapshot();
+      const sumQuery = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'sum',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: '',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(sumQuery)).toMatchSnapshot();
+    });
+
+    it('single avg gauge with where', async () => {
+      const query = await renderChartConfig(
+        {
+          select: [
+            {
+              aggFn: 'avg',
+              metricName: 'test.cpu',
+              metricType: MetricsDataType.Gauge,
+              valueExpression: 'Value',
+            },
+          ],
+          from: metricSource.from,
+          where: `ResourceAttributes['host'] = 'host1'`,
+          whereLanguage: 'sql',
+          metricTables: {
+            sum: DEFAULT_METRICS_TABLE.SUM,
+            gauge: DEFAULT_METRICS_TABLE.GAUGE,
+            histogram: DEFAULT_METRICS_TABLE.HISTOGRAM,
+          },
+          dateRange: [new Date(now), new Date(now + ms('10m'))],
+          granularity: '5 minute',
+          timestampValueExpression: metricSource.timestampValueExpression,
+          connection: connection.id,
+        },
+        metadata,
+      );
+      expect(await queryData(query)).toMatchSnapshot();
     });
 
-    // FIXME: gauge avg doesn't work as expected (it should pull the average of the last value)
-    // in this case
-    // (6.25 + 4) / 2, (80 + 4) / 2
-    it('single avg gauge', async () => {
+    it('single avg gauge with group-by', async () => {
       const query = await renderChartConfig(
         {
           select: [
@@ -367,11 +467,13 @@ describe('renderChartConfig', () => {
           },
           dateRange: [new Date(now), new Date(now + ms('10m'))],
           granularity: '5 minute',
+          groupBy: `ResourceAttributes['host']`,
           timestampValueExpression: metricSource.timestampValueExpression,
           connection: connection.id,
         },
         metadata,
       );
+      expect(await queryData(query)).toMatchSnapshot();
     });
 
     it('single sum rate', async () => {

packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -0,0 +1,29 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
+"WITH Bucketed AS (
+            SELECT
+              toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`,
+              ScopeAttributes,
+              ResourceAttributes,
+              Attributes,
+              cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM default.otel_metrics_gauge
+            WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'nodejs.event_loop.utilization'))
+            GROUP BY ScopeAttributes, ResourceAttributes, Attributes, __hdx_time_bucket2
+            ORDER BY AttributesHash, __hdx_time_bucket2
+          ) SELECT quantile(0.95)(toFloat64OrNull(toString(LastValue))),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))
+      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))
+      STEP 60 LIMIT 10"
+`;

packages/common-utils/src/__tests__/renderChartConfig.test.ts

@@ -17,7 +17,7 @@ describe('renderChartConfig', () => {
         { name: 'timestamp', type: 'DateTime' },
         { name: 'value', type: 'Float64' },
       ]),
-      getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue({}),
+      getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
       getColumn: jest.fn().mockResolvedValue({ type: 'DateTime' }),
     } as unknown as Metadata;
   });
@@ -57,16 +57,7 @@ describe('renderChartConfig', () => {
 
     const generatedSql = await renderChartConfig(config, mockMetadata);
     const actual = parameterizedQueryToSql(generatedSql);
-    expect(actual).toBe(
-      'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
-        ' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' +
-        " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" +
-        ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
-        ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' +
-        '      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' +
-        '      STEP 60' +
-        ' LIMIT 10',
-    );
+    expect(actual).toMatchSnapshot();
   });
 
   it('should generate sql for a single sum metric', async () => {

packages/common-utils/src/renderChartConfig.ts

@@ -38,6 +38,7 @@ function determineTableName(select: SelectSQLStatement): string {
   return '';
 }
 
+const DEFAULT_METRIC_TABLE_TIME_COLUMN = 'TimeUnix';
 export const FIXED_TIME_BUCKET_EXPR_ALIAS = '__hdx_time_bucket';
 
 export function isUsingGroupBy(
@@ -784,9 +785,10 @@ function renderFill(
   return undefined;
 }
 
-function translateMetricChartConfig(
+async function translateMetricChartConfig(
   chartConfig: ChartConfigWithOptDateRange,
-): ChartConfigWithOptDateRangeEx {
+  metadata: Metadata,
+): Promise<ChartConfigWithOptDateRangeEx> {
   const metricTables = chartConfig.metricTables;
   if (!metricTables) {
     return chartConfig;
@@ -800,20 +802,74 @@ function translateMetricChartConfig(
 
   const { metricType, metricName, ..._select } = select[0]; // Initial impl only supports one metric select per chart config
   if (metricType === MetricsDataType.Gauge && metricName) {
+    const timeBucketCol = '__hdx_time_bucket2';
+    const timeExpr = timeBucketExpr({
+      interval: chartConfig.granularity || 'auto',
+      timestampValueExpression:
+        chartConfig.timestampValueExpression ||
+        DEFAULT_METRIC_TABLE_TIME_COLUMN,
+      dateRange: chartConfig.dateRange,
+      alias: timeBucketCol,
+    });
+
+    const where = await renderWhere(
+      {
+        ...chartConfig,
+        from: {
+          ...from,
+          tableName: metricTables[MetricsDataType.Gauge],
+        },
+        filters: [
+          {
+            type: 'sql',
+            condition: `MetricName = '${metricName}'`,
+          },
+        ],
+      },
+      metadata,
+    );
+
     return {
       ...restChartConfig,
+      with: [
+        {
+          name: 'Bucketed',
+          sql: chSql`
+            SELECT
+              ${timeExpr},
+              ScopeAttributes,
+              ResourceAttributes,
+              Attributes,
+              cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })}
+            WHERE ${where}
+            GROUP BY ScopeAttributes, ResourceAttributes, Attributes, ${timeBucketCol}
+            ORDER BY AttributesHash, ${timeBucketCol}
+          `,
+        },
+      ],
       select: [
         {
           ..._select,
-          valueExpression: 'Value',
+          valueExpression: 'LastValue',
         },
       ],
       from: {
-        ...from,
-        tableName: metricTables[MetricsDataType.Gauge],
+        databaseName: '',
+        tableName: 'Bucketed',
       },
-      where: `MetricName = '${metricName}'`,
-      whereLanguage: 'sql',
+      timestampValueExpression: timeBucketCol,
     };
   } else if (metricType === MetricsDataType.Sum && metricName) {
     return {
@@ -921,7 +977,7 @@ export async function renderChartConfig(
   // but goes through the same generation process
   const chartConfig =
     rawChartConfig.metricTables != null
-      ? translateMetricChartConfig(rawChartConfig)
+      ? await translateMetricChartConfig(rawChartConfig, metadata)
       : rawChartConfig;
 
   const withClauses = renderWith(chartConfig, metadata);