feat: supporting quantile histogram metrics (#635)

dhable · web-flow · commit e80630c1075d · 2025-02-27T16:55:36.000Z
Additional `renderChartConfig` support to transform a histogram select into the correct SQL syntax to generate a chart. For parity with v1, this query only handles quantile queries. <img width="1939" alt="Screenshot 2025-02-26 at 12 58 55 PM" src="https://github.com/user-attachments/assets/1126ac6c-c431-4d89-92d7-9df1e49e25cf" /> <img width="1960" alt="Screenshot 2025-02-26 at 3 11 07 PM" src="https://github.com/user-attachments/assets/e4fa09bf-1e27-4a90-ad25-6c6cb2890877" /> Ref: HDX-1339
diff --git a/.changeset/odd-hats-wash.md b/.changeset/odd-hats-wash.md
@@ -0,0 +1,5 @@
+---
+"@hyperdx/common-utils": minor
+---
+
+Add chart support for querying OTEL histogram metric table
diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts
@@ -60,8 +60,12 @@ describe('renderChartConfig', () => {
     expect(actual).toBe(
       'SELECT quantile(0.95)(toFloat64OrNull(toString(Value))),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
         ' FROM default.otel_metrics_gauge WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND' +
-        " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` " +
-        'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket` LIMIT 10',
+        " (MetricName = 'nodejs.event_loop.utilization') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`" +
+        ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS `__hdx_time_bucket`' +
+        ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 1 minute))\n' +
+        '      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 1 minute))\n' +
+        '      STEP 60' +
+        ' LIMIT 10',
     );
   });
 
@@ -93,7 +97,7 @@ describe('renderChartConfig', () => {
       whereLanguage: 'sql',
       timestampValueExpression: 'TimeUnix',
       dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
-      granularity: '5 minutes',
+      granularity: '5 minute',
       limit: { limit: 10 },
     };
 
@@ -112,12 +116,84 @@ describe('renderChartConfig', () => {
         '                FROM default.otel_metrics_sum\n' +
         "                WHERE MetricName = 'db.client.connections.usage'\n" +
         '                ORDER BY AttributesHash, TimeUnix ASC\n' +
-        '            ) )SELECT avg(\n' +
+        '            ) ) SELECT avg(\n' +
         '      toFloat64OrNull(toString(Rate))\n' +
-        '    ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` ' +
-        'FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) ' +
-        "AND (MetricName = 'db.client.connections.usage') GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` " +
-        'ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minutes) AS `__hdx_time_bucket` LIMIT 10',
+        '    ),toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
+        ' FROM RawSum WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' +
+        ' GROUP BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
+        ' ORDER BY toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS `__hdx_time_bucket`' +
+        ' WITH FILL FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute))\n' +
+        '      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute))\n' +
+        '      STEP 300' +
+        ' LIMIT 10',
+    );
+  });
+
+  it('should generate sql for a single histogram metric', async () => {
+    const config: ChartConfigWithOptDateRange = {
+      displayType: DisplayType.Line,
+      connection: 'test-connection',
+      // metricTables is added from the Source object via spread operator
+      metricTables: {
+        gauge: 'otel_metrics_gauge',
+        histogram: 'otel_metrics_histogram',
+        sum: 'otel_metrics_sum',
+      },
+      from: {
+        databaseName: 'default',
+        tableName: '',
+      },
+      select: [
+        {
+          aggFn: 'quantile',
+          level: 0.5,
+          valueExpression: 'Value',
+          metricName: 'http.server.duration',
+          metricType: MetricsDataType.Histogram,
+        },
+      ],
+      where: '',
+      whereLanguage: 'sql',
+      timestampValueExpression: 'TimeUnix',
+      dateRange: [new Date('2025-02-12'), new Date('2025-12-14')],
+      limit: { limit: 10 },
+    };
+
+    const generatedSql = await renderChartConfig(config, mockMetadata);
+    const actual = parameterizedQueryToSql(generatedSql);
+    expect(actual).toBe(
+      'WITH HistRate AS (SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,\n' +
+        '            any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,\n' +
+        '            any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,\n' +
+        '            IF(AggregationTemporality = 1,\n' +
+        '               BucketCounts,\n' +
+        '               IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,\n' +
+        '                  arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),\n' +
+        '                  BucketCounts)) as BucketRates\n' +
+        '          FROM (\n' +
+        '            SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,\n' +
+        '                   length(BucketCounts) as CountLength\n' +
+        '            FROM default.otel_metrics_histogram)\n' +
+        "            WHERE MetricName = 'http.server.duration'\n " +
+        '           ORDER BY Attributes, TimeUnix ASC\n' +
+        '          ),RawHist AS (\n' +
+        '            SELECT *, toUInt64( 0.5 * arraySum(BucketRates)) AS Rank,\n' +
+        '                   arrayCumSum(BucketRates) as CumRates,\n' +
+        '                   arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,\n' +
+        '                   IF(BucketLowIdx = length(BucketRates),\n' +
+        '                      ExplicitBounds[length(ExplicitBounds)],  -- if the low bound is the last bucket, use the last bound value\n' +
+        '                      IF(BucketLowIdx > 1, -- indexes are 1-based\n' +
+        '                         ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) *\n' +
+        '                         intDivOrZero(\n' +
+        '                             Rank - CumRates[BucketLowIdx - 1],\n' +
+        '                             CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]),\n' +
+        '                    arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate\n' +
+        '            FROM HistRate) SELECT sum(\n' +
+        '      toFloat64OrNull(toString(Rate))\n' +
+        '    )' +
+        ' FROM RawHist' +
+        ' WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000))' +
+        ' LIMIT 10',
     );
   });
 });
diff --git a/packages/common-utils/src/clickhouse.ts b/packages/common-utils/src/clickhouse.ts
@@ -186,7 +186,11 @@ export const concatChSql = (sep: string, ...args: (ChSql | ChSql[])[]) => {
         }
 
         acc.sql +=
-          (acc.sql.length > 0 ? sep : '') + arg.map(a => a.sql).join(sep);
+          (acc.sql.length > 0 ? sep : '') +
+          arg
+            .map(a => a.sql)
+            .filter(Boolean) // skip empty string expressions
+            .join(sep);
         acc.params = arg.reduce((acc, a) => {
           Object.assign(acc, a.params);
           return acc;
diff --git a/packages/common-utils/src/renderChartConfig.ts b/packages/common-utils/src/renderChartConfig.ts
@@ -292,7 +292,7 @@ const aggFnExpr = ({
 
 async function renderSelectList(
   selectList: SelectList,
-  chartConfig: ChartConfigWithOptDateRateAndCte,
+  chartConfig: ChartConfigWithOptDateRangeEx,
   metadata: Metadata,
 ) {
   if (typeof selectList === 'string') {
@@ -467,7 +467,7 @@ async function timeFilterExpr({
 }
 
 async function renderSelect(
-  chartConfig: ChartConfigWithOptDateRateAndCte,
+  chartConfig: ChartConfigWithOptDateRangeEx,
   metadata: Metadata,
 ): Promise<ChSql> {
   /**
@@ -565,7 +565,7 @@ async function renderWhereExpression({
 }
 
 async function renderWhere(
-  chartConfig: ChartConfigWithOptDateRateAndCte,
+  chartConfig: ChartConfigWithOptDateRangeEx,
   metadata: Metadata,
 ): Promise<ChSql> {
   let whereSearchCondition: ChSql | [] = [];
@@ -729,28 +729,64 @@ function renderLimit(
 
 // CTE (Common Table Expressions) isn't exported at this time. It's only used internally
 // for metric SQL generation.
-type ChartConfigWithOptDateRateAndCte = ChartConfigWithOptDateRange & {
+type ChartConfigWithOptDateRangeEx = ChartConfigWithOptDateRange & {
   with?: { name: string; sql: ChSql }[];
 };
 
 function renderWith(
-  chartConfig: ChartConfigWithOptDateRateAndCte,
+  chartConfig: ChartConfigWithOptDateRangeEx,
   metadata: Metadata,
 ): ChSql | undefined {
   const { with: withClauses } = chartConfig;
   if (withClauses) {
     return concatChSql(
-      '',
-      withClauses.map(clause => chSql`WITH ${clause.name} AS (${clause.sql})`),
+      ',',
+      withClauses.map(clause => chSql`${clause.name} AS (${clause.sql})`),
     );
   }
 
   return undefined;
 }
 
+function intervalToSeconds(interval: SQLInterval): number {
+  // Parse interval string like "15 second" into number of seconds
+  const [amount, unit] = interval.split(' ');
+  const value = parseInt(amount, 10);
+  switch (unit) {
+    case 'second':
+      return value;
+    case 'minute':
+      return value * 60;
+    case 'hour':
+      return value * 60 * 60;
+    case 'day':
+      return value * 24 * 60 * 60;
+    default:
+      throw new Error(`Invalid interval unit ${unit} in interval ${interval}`);
+  }
+}
+
+function renderFill(
+  chartConfig: ChartConfigWithOptDateRangeEx,
+): ChSql | undefined {
+  const { granularity, dateRange } = chartConfig;
+  if (dateRange && granularity && granularity !== 'auto') {
+    const [start, end] = dateRange;
+    const step = intervalToSeconds(granularity);
+
+    return concatChSql(' ', [
+      chSql`FROM toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: start.getTime() }}), INTERVAL ${granularity}))
+      TO toUnixTimestamp(toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: end.getTime() }}), INTERVAL ${granularity}))
+      STEP ${{ Int32: step }}`,
+    ]);
+  }
+
+  return undefined;
+}
+
 function translateMetricChartConfig(
   chartConfig: ChartConfigWithOptDateRange,
-): ChartConfigWithOptDateRateAndCte {
+): ChartConfigWithOptDateRangeEx {
   const metricTables = chartConfig.metricTables;
   if (!metricTables) {
     return chartConfig;
@@ -810,8 +846,67 @@ function translateMetricChartConfig(
         databaseName: '',
         tableName: 'RawSum',
       },
-      where: `MetricName = '${metricName}'`,
-      whereLanguage: 'sql',
+    };
+  } else if (metricType === MetricsDataType.Histogram && metricName) {
+    // histograms are only valid for quantile selections
+    const { aggFn, level, ..._selectRest } = _select as {
+      aggFn: string;
+      level?: number;
+    };
+
+    if (aggFn !== 'quantile' || level == null) {
+      throw new Error('quantile must be specified for histogram metrics');
+    }
+
+    return {
+      ...restChartConfig,
+      with: [
+        {
+          name: 'HistRate',
+          sql: chSql`SELECT *, any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,
+            any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,
+            any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,
+            IF(AggregationTemporality = 1,
+               BucketCounts,
+               IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,
+                  arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),
+                  BucketCounts)) as BucketRates
+          FROM (
+            SELECT *, cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
+                   length(BucketCounts) as CountLength
+            FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Histogram] } })})
+            WHERE MetricName = '${metricName}'
+            ORDER BY Attributes, TimeUnix ASC
+          `,
+        },
+        {
+          name: 'RawHist',
+          sql: chSql`
+            SELECT *, toUInt64( ${{ Float64: level }} * arraySum(BucketRates)) AS Rank,
+                   arrayCumSum(BucketRates) as CumRates,
+                   arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,
+                   IF(BucketLowIdx = length(BucketRates),
+                      ExplicitBounds[length(ExplicitBounds)],  -- if the low bound is the last bucket, use the last bound value
+                      IF(BucketLowIdx > 1, -- indexes are 1-based
+                         ExplicitBounds[BucketLowIdx] + (ExplicitBounds[BucketLowIdx + 1] - ExplicitBounds[BucketLowIdx]) *
+                         intDivOrZero(
+                             Rank - CumRates[BucketLowIdx - 1],
+                             CumRates[BucketLowIdx] - CumRates[BucketLowIdx - 1]),
+                    arrayElement(ExplicitBounds, BucketLowIdx + 1) * intDivOrZero(Rank, CumRates[BucketLowIdx]))) as Rate
+            FROM HistRate`,
+        },
+      ],
+      select: [
+        {
+          ..._selectRest,
+          aggFn: 'sum',
+          valueExpression: 'Rate',
+        },
+      ],
+      from: {
+        databaseName: '',
+        tableName: 'RawHist',
+      },
     };
   }
 
@@ -835,15 +930,19 @@ export async function renderChartConfig(
   const where = await renderWhere(chartConfig, metadata);
   const groupBy = await renderGroupBy(chartConfig, metadata);
   const orderBy = renderOrderBy(chartConfig);
+  const fill = renderFill(chartConfig);
   const limit = renderLimit(chartConfig);
 
-  return chSql`${
-    withClauses?.sql ? chSql`${withClauses}` : ''
-  }SELECT ${select} FROM ${from} ${where?.sql ? chSql`WHERE ${where}` : ''} ${
-    groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''
-  } ${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''} ${
-    limit?.sql ? chSql`LIMIT ${limit}` : ''
-  }`;
+  return concatChSql(' ', [
+    chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`,
+    chSql`SELECT ${select}`,
+    chSql`FROM ${from}`,
+    chSql`${where.sql ? chSql`WHERE ${where}` : ''}`,
+    chSql`${groupBy?.sql ? chSql`GROUP BY ${groupBy}` : ''}`,
+    chSql`${orderBy?.sql ? chSql`ORDER BY ${orderBy}` : ''}`,
+    chSql`${fill?.sql ? chSql`WITH FILL ${fill}` : ''}`,
+    chSql`${limit?.sql ? chSql`LIMIT ${limit}` : ''}`,
+  ]);
 }
 
 // EditForm -> translateToQueriedChartConfig -> QueriedChartConfig

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@hyperdx/common-utils": minor
 +---
++
 +Add chart support for querying OTEL histogram metric table