fix: autocomplete for lucene column values

.changeset/fifty-pugs-nail.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+fix: autocomplete for key-values complete for v2 lucene

packages/app/src/NamespaceDetailsSidePanel.tsx

@@ -278,7 +278,7 @@ export default function NamespaceDetailsSidePanel({
 
   const { data: logServiceNames } = useGetKeyValues(
     {
-      chartConfig: {
+      chartConfigs: {
         from: logSource.from,
         where: `${logSource?.resourceAttributesExpression}.k8s.namespace.name:"${namespaceName}"`,
         whereLanguage: 'lucene',

packages/app/src/NodeDetailsSidePanel.tsx

@@ -297,7 +297,7 @@ export default function NodeDetailsSidePanel({
 
   const { data: logServiceNames } = useGetKeyValues(
     {
-      chartConfig: {
+      chartConfigs: {
         from: logSource.from,
         where: `${logSource?.resourceAttributesExpression}.k8s.node.name:"${nodeName}"`,
         whereLanguage: 'lucene',

packages/app/src/PodDetailsSidePanel.tsx

@@ -280,7 +280,7 @@ export default function PodDetailsSidePanel({
 
   const { data: logServiceNames } = useGetKeyValues(
     {
-      chartConfig: {
+      chartConfigs: {
         from: logSource.from,
         where: `${logSource?.resourceAttributesExpression}.k8s.pod.name:"${podName}"`,
         whereLanguage: 'lucene',

packages/app/src/SearchInputV2.tsx

@@ -1,11 +1,27 @@
-import { useEffect, useMemo, useRef, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { useController, UseControllerProps } from 'react-hook-form';
 import { useHotkeys } from 'react-hotkeys-hook';
-import { TableConnection } from '@hyperdx/common-utils/dist/metadata';
+import { Field, TableConnection } from '@hyperdx/common-utils/dist/metadata';
 import { genEnglishExplanation } from '@hyperdx/common-utils/dist/queryParser';
 
 import AutocompleteInput from '@/AutocompleteInput';
-import { useAllFields } from '@/hooks/useMetadata';
+
+import {
+  ILanguageFormatter,
+  useAutoCompleteOptions,
+} from './hooks/useAutoCompleteOptions';
+
+export class LuceneLanguageFormatter implements ILanguageFormatter {
+  formatFieldValue(f: Field): string {
+    return f.path.join('.');
+  }
+  formatFieldLabel(f: Field): string {
+    return `${f.path.join('.')} (${f.jsType})`;
+  }
+  formatKeyValPair(key: string, value: string): string {
+    return `${key}:"${value}"`;
+  }
+}
 
 export default function SearchInputV2({
   tableConnections,
@@ -34,31 +50,17 @@ export default function SearchInputV2({
   } = useController(props);
 
   const ref = useRef<HTMLInputElement>(null);
-
-  const { data: fields } = useAllFields(tableConnections ?? [], {
-    enabled:
-      !!tableConnections &&
-      (Array.isArray(tableConnections) ? tableConnections.length > 0 : true),
-  });
-
-  const autoCompleteOptions = useMemo(() => {
-    const _columns = (fields ?? []).filter(c => c.jsType !== null);
-    const baseOptions = _columns.map(c => ({
-      value: c.path.join('.'),
-      label: `${c.path.join('.')} (${c.jsType})`,
-    }));
-
-    const suggestionOptions =
-      additionalSuggestions?.map(column => ({
-        value: column,
-        label: column,
-      })) ?? [];
-
-    return [...baseOptions, ...suggestionOptions];
-  }, [fields, additionalSuggestions]);
-
   const [parsedEnglishQuery, setParsedEnglishQuery] = useState<string>('');
 
+  const autoCompleteOptions = useAutoCompleteOptions(
+    new LuceneLanguageFormatter(),
+    value,
+    {
+      tableConnections,
+      additionalSuggestions,
+    },
+  );
+
   useEffect(() => {
     genEnglishExplanation(value).then(q => {
       setParsedEnglishQuery(q);

packages/app/src/components/DBSearchPageFilters.tsx

@@ -380,7 +380,7 @@ export const DBSearchPageFilters = ({
     isLoading: isFacetsLoading,
     isFetching: isFacetsFetching,
   } = useGetKeyValues({
-    chartConfig: { ...chartConfig, dateRange },
+    chartConfigs: { ...chartConfig, dateRange },
     keys: datum,
   });
 

packages/app/src/components/MetricNameSelect.tsx

@@ -42,19 +42,19 @@ function useMetricNames(metricSource: TSource) {
   }, [metricSource]);
 
   const { data: gaugeMetrics } = useGetKeyValues({
-    chartConfig: gaugeConfig,
+    chartConfigs: gaugeConfig,
     keys: ['MetricName'],
     limit: MAX_METRIC_NAME_OPTIONS,
     disableRowLimit: true,
   });
   // const { data: histogramMetrics } = useGetKeyValues({
-  //   chartConfig: histogramConfig,
+  //   chartConfigs: histogramConfig,
   //   keys: ['MetricName'],
   //   limit: MAX_METRIC_NAME_OPTIONS,
   //   disableRowLimit: true,
   // });
   const { data: sumMetrics } = useGetKeyValues({
-    chartConfig: sumConfig,
+    chartConfigs: sumConfig,
     keys: ['MetricName'],
     limit: MAX_METRIC_NAME_OPTIONS,
     disableRowLimit: true,

packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx

@@ -0,0 +1,210 @@
+import { JSDataType } from '@hyperdx/common-utils/dist/clickhouse';
+import { Field } from '@hyperdx/common-utils/dist/metadata';
+import { renderHook } from '@testing-library/react';
+
+import { LuceneLanguageFormatter } from '../../SearchInputV2';
+import { useAutoCompleteOptions } from '../useAutoCompleteOptions';
+import { useAllFields, useGetKeyValues } from '../useMetadata';
+
+if (!globalThis.structuredClone) {
+  globalThis.structuredClone = (obj: any) => {
+    return JSON.parse(JSON.stringify(obj));
+  };
+}
+
+// Mock dependencies
+jest.mock('../useMetadata', () => ({
+  ...jest.requireActual('../useMetadata.tsx'),
+  useAllFields: jest.fn(),
+  useGetKeyValues: jest.fn(),
+}));
+
+const luceneFormatter = new LuceneLanguageFormatter();
+
+const mockFields: Field[] = [
+  {
+    path: ['ResourceAttributes'],
+    jsType: JSDataType.Map,
+    type: 'map',
+  },
+  {
+    path: ['ResourceAttributes', 'service.name'],
+    jsType: JSDataType.String,
+    type: 'string',
+  },
+  {
+    path: ['TraceAttributes', 'trace.id'],
+    jsType: JSDataType.String,
+    type: 'string',
+  },
+];
+
+const mockTableConnections = [
+  {
+    databaseName: 'test_db',
+    tableName: 'traces',
+    connectionId: 'conn1',
+  },
+];
+
+describe('useAutoCompleteOptions', () => {
+  beforeEach(() => {
+    // Reset mocks before each test
+    jest.clearAllMocks();
+
+    // Setup default mock implementations
+    (useAllFields as jest.Mock).mockReturnValue({
+      data: mockFields,
+    });
+
+    (useGetKeyValues as jest.Mock).mockReturnValue({
+      data: null,
+    });
+  });
+
+  it('should return field options with correct lucene formatting', () => {
+    const { result } = renderHook(() =>
+      useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
+        tableConnections: mockTableConnections,
+      }),
+    );
+
+    expect(result.current).toEqual([
+      {
+        value: 'ResourceAttributes',
+        label: 'ResourceAttributes (map)',
+      },
+      {
+        value: 'ResourceAttributes.service.name',
+        label: 'ResourceAttributes.service.name (string)',
+      },
+      {
+        value: 'TraceAttributes.trace.id',
+        label: 'TraceAttributes.trace.id (string)',
+      },
+    ]);
+  });
+
+  it('should return key value options with correct lucene formatting', () => {
+    const mockKeyValues = [
+      {
+        key: 'ResourceAttributes.service.name',
+        value: ['frontend', 'backend'],
+      },
+    ];
+
+    (useGetKeyValues as jest.Mock).mockReturnValue({
+      data: mockKeyValues,
+    });
+
+    const { result } = renderHook(() =>
+      useAutoCompleteOptions(
+        luceneFormatter,
+        'ResourceAttributes.service.name',
+        {
+          tableConnections: mockTableConnections,
+        },
+      ),
+    );
+
+    expect(result.current).toEqual([
+      {
+        value: 'ResourceAttributes',
+        label: 'ResourceAttributes (map)',
+      },
+      {
+        value: 'ResourceAttributes.service.name',
+        label: 'ResourceAttributes.service.name (string)',
+      },
+      {
+        value: 'TraceAttributes.trace.id',
+        label: 'TraceAttributes.trace.id (string)',
+      },
+      {
+        value: 'ResourceAttributes.service.name:"frontend"',
+        label: 'ResourceAttributes.service.name:"frontend"',
+      },
+      {
+        value: 'ResourceAttributes.service.name:"backend"',
+        label: 'ResourceAttributes.service.name:"backend"',
+      },
+    ]);
+  });
+
+  // TODO: Does this test case need to be removed after HDX-1548?
+  it('should handle nested key value options', () => {
+    const mockKeyValues = [
+      {
+        key: 'ResourceAttributes',
+        value: [
+          {
+            'service.name': 'frontend',
+            'deployment.environment': 'production',
+          },
+        ],
+      },
+    ];
+
+    (useGetKeyValues as jest.Mock).mockReturnValue({
+      data: mockKeyValues,
+    });
+
+    const { result } = renderHook(() =>
+      useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
+        tableConnections: mockTableConnections,
+      }),
+    );
+
+    //console.log(result.current);
+    expect(result.current).toEqual([
+      {
+        value: 'ResourceAttributes',
+        label: 'ResourceAttributes (map)',
+      },
+      {
+        value: 'ResourceAttributes.service.name',
+        label: 'ResourceAttributes.service.name (string)',
+      },
+      {
+        value: 'TraceAttributes.trace.id',
+        label: 'TraceAttributes.trace.id (string)',
+      },
+      {
+        value: 'ResourceAttributes.service.name:"frontend"',
+        label: 'ResourceAttributes.service.name:"frontend"',
+      },
+      {
+        value: 'ResourceAttributes.deployment.environment:"production"',
+        label: 'ResourceAttributes.deployment.environment:"production"',
+      },
+    ]);
+  });
+
+  it('should handle additional suggestions', () => {
+    const { result } = renderHook(() =>
+      useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
+        tableConnections: mockTableConnections,
+        additionalSuggestions: ['custom.field'],
+      }),
+    );
+
+    expect(result.current).toEqual([
+      {
+        value: 'ResourceAttributes',
+        label: 'ResourceAttributes (map)',
+      },
+      {
+        value: 'ResourceAttributes.service.name',
+        label: 'ResourceAttributes.service.name (string)',
+      },
+      {
+        value: 'TraceAttributes.trace.id',
+        label: 'TraceAttributes.trace.id (string)',
+      },
+      {
+        value: 'custom.field',
+        label: 'custom.field',
+      },
+    ]);
+  });
+});