added fix for jobs and cache Support for workbench ,MDS support (#1739) (#1784)

* cacahe updated to support MDS-client and added MDS support



* addressed PR comments, added fix for jobs and cache



* updated the tests, PR comments



* addressed comments, fixed loading of flint datasources



* added fix for cache and cahnged router



* fixed linter



---------



(cherry picked from commit 868ddcb)

Signed-off-by: sumukhswamy <[email protected]>
Signed-off-by: Sumukh Swamy <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: opensearch-trigger-bot[bot]

common/types/data_connections.ts

@@ -121,6 +121,7 @@ export interface CachedDataSource {
   lastUpdated: string; // date string in UTC format
   status: CachedDataSourceStatus;
   databases: CachedDatabase[];
+  dataSourceMDSId?: string;
 }
 
 export interface DataSourceCacheData {
@@ -143,6 +144,7 @@ export interface CachedAccelerationByDataSource {
   accelerations: CachedAcceleration[];
   lastUpdated: string; // date string in UTC format
   status: CachedDataSourceStatus;
+  dataSourceMDSId?: string;
 }
 
 export interface AccelerationsCacheData {
@@ -240,6 +242,13 @@ export interface CreateAccelerationForm {
 
 export interface LoadCachehookOutput {
   loadStatus: DirectQueryLoadingStatus;
-  startLoading: (dataSourceName: string, databaseName?: string, tableName?: string) => void;
+  startLoading: (params: StartLoadingParams) => void;
   stopLoading: () => void;
 }
+
+export interface StartLoadingParams {
+  dataSourceName: string;
+  dataSourceMDSId?: string;
+  databaseName?: string;
+  tableName?: string;
+}

common/utils/shared.ts

@@ -18,6 +18,9 @@ export function get<T = unknown>(obj: Record<string, any>, path: string, default
 }
 
 export function addBackticksIfNeeded(input: string): string {
+  if (input === undefined) {
+    return '';
+  }
   // Check if the string already has backticks
   if (input.startsWith('`') && input.endsWith('`')) {
     return input; // Return the string as it is

public/components/datasources/components/manage/accelerations/acceleration_details_flyout.tsx

@@ -16,24 +16,25 @@ import {
   EuiText,
 } from '@elastic/eui';
 import React, { useEffect, useState } from 'react';
+import { OpenSearchDashboardsResponse } from '../../../../../../../../src/core/server/http/router';
+import { CachedAcceleration } from '../../../../../../common/types/data_connections';
+import { coreRefs } from '../../../../../framework/core_refs';
+import { AccelerationActionOverlay } from './acceleration_action_overlay';
+import { useAccelerationOperation } from './acceleration_operation';
 import { AccelerationDetailsTab } from './flyout_modules/acceleration_details_tab';
 import { AccelerationSchemaTab } from './flyout_modules/accelerations_schema_tab';
 import {
-  onDiscoverIconClick,
   AccelerationActionType,
   getAccelerationName,
+  onDiscoverIconClick,
 } from './utils/acceleration_utils';
-import { coreRefs } from '../../../../../framework/core_refs';
-import { OpenSearchDashboardsResponse } from '../../../../../../../../src/core/server/http/router';
-import { CachedAcceleration } from '../../../../../../common/types/data_connections';
-import { useAccelerationOperation } from './acceleration_operation';
-import { AccelerationActionOverlay } from './acceleration_action_overlay';
 
 export interface AccelerationDetailsFlyoutProps {
   acceleration: CachedAcceleration;
   dataSourceName: string;
   resetFlyout: () => void;
   handleRefresh?: () => void;
+  dataSourceMDSId?: string;
 }
 
 const getMappings = (index: string): Promise<OpenSearchDashboardsResponse> | undefined => {

public/components/datasources/components/manage/accelerations/acceleration_table.tsx

@@ -18,27 +18,29 @@ import {
   EuiText,
 } from '@elastic/eui';
 import React, { useCallback, useEffect, useState } from 'react';
-import {
-  onDiscoverIconClick,
-  AccelerationStatus,
-  ACC_LOADING_MSG,
-  ACC_PANEL_TITLE,
-  ACC_PANEL_DESC,
-  getAccelerationName,
-  AccelerationActionType,
-  CreateAccelerationFlyoutButton,
-} from './utils/acceleration_utils';
-import { getRenderAccelerationDetailsFlyout } from '../../../../../plugin';
-import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager';
 import {
   CachedAcceleration,
   CachedDataSourceStatus,
 } from '../../../../../../common/types/data_connections';
 import { DirectQueryLoadingStatus } from '../../../../../../common/types/explorer';
-import { AccelerationActionOverlay } from './acceleration_action_overlay';
+import { CatalogCacheManager } from '../../../../../framework/catalog_cache/cache_manager';
+import {
+  getRenderAccelerationDetailsFlyout,
+  getRenderCreateAccelerationFlyout,
+} from '../../../../../plugin';
 import { isCatalogCacheFetching } from '../associated_objects/utils/associated_objects_tab_utils';
-import { getRenderCreateAccelerationFlyout } from '../../../../../plugin';
+import { AccelerationActionOverlay } from './acceleration_action_overlay';
 import { useAccelerationOperation } from './acceleration_operation';
+import {
+  ACC_LOADING_MSG,
+  ACC_PANEL_DESC,
+  ACC_PANEL_TITLE,
+  AccelerationActionType,
+  AccelerationStatus,
+  CreateAccelerationFlyoutButton,
+  getAccelerationName,
+  onDiscoverIconClick,
+} from './utils/acceleration_utils';
 
 interface AccelerationTableProps {
   dataSourceName: string;
@@ -108,7 +110,7 @@ export const AccelerationTable = ({
       !isCatalogCacheFetching(accelerationsLoadStatus)
     ) {
       setIsRefreshing(true);
-      startLoadingAccelerations(dataSourceName);
+      startLoadingAccelerations({ dataSourceName });
     } else {
       setAccelerations(cachedDataSource.accelerations);
       setUpdatedTime(cachedDataSource.lastUpdated);

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/create/create_acceleration.tsx

@@ -45,6 +45,7 @@ export interface CreateAccelerationProps {
   resetFlyout: () => void;
   databaseName?: string;
   tableName?: string;
+  dataSourceMDSId?: string;
   refreshHandler?: () => void;
 }
 
@@ -53,6 +54,7 @@ export const CreateAcceleration = ({
   resetFlyout,
   databaseName,
   tableName,
+  dataSourceMDSId,
   refreshHandler,
 }: CreateAccelerationProps) => {
   const { setToast } = useToast();
@@ -141,12 +143,22 @@ export const CreateAcceleration = ({
     if (dataTable !== '') {
       setTableFieldsLoading(true);
       try {
-        const cachedTable = CatalogCacheManager.getTable(dataSource, database, dataTable);
+        const cachedTable = CatalogCacheManager.getTable(
+          dataSource,
+          database,
+          dataTable,
+          dataSourceMDSId
+        );
         if (cachedTable.columns) {
           loadColumnsToAccelerationForm(cachedTable);
           setTableFieldsLoading(false);
         } else {
-          startLoading(dataSource, database, dataTable);
+          startLoading({
+            dataSourceName: dataSource,
+            dataSourceMDSId,
+            databaseName: database,
+            tableName: dataTable,
+          });
         }
       } catch (error) {
         setToast('Your cache is outdated, refresh databases and tables', 'warning');
@@ -173,7 +185,8 @@ export const CreateAcceleration = ({
         cachedTable = CatalogCacheManager.getTable(
           accelerationFormData.dataSource,
           accelerationFormData.database,
-          accelerationFormData.dataTable
+          accelerationFormData.dataTable,
+          dataSourceMDSId
         );
       } catch (error) {
         setToast('Your cache is outdated, refresh databases and tables', 'warning');
@@ -218,6 +231,7 @@ export const CreateAcceleration = ({
               selectedDatasource={selectedDatasource}
               dataSourcesPreselected={dataSourcesPreselected}
               tableFieldsLoading={tableFieldsLoading}
+              dataSourceMDSId={dataSourceMDSId}
             />
             <EuiSpacer size="xxl" />
             <IndexTypeSelector
@@ -235,6 +249,7 @@ export const CreateAcceleration = ({
               accelerationFormData={accelerationFormData}
               setAccelerationFormData={setAccelerationFormData}
               tableFieldsLoading={tableFieldsLoading}
+              dataSourceMDSId={dataSourceMDSId}
             />
             <EuiSpacer size="xxl" />
             <IndexAdvancedSettings

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_databases.tsx

@@ -42,7 +42,7 @@ export const SelectorLoadDatabases = ({
 
   const onClickRefreshDatabases = () => {
     setIsLoading(true);
-    startDatabasesLoading(dataSourceName);
+    startDatabasesLoading({ dataSourceName });
   };
 
   useEffect(() => {

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/selector_helpers/load_objects.tsx

@@ -65,8 +65,8 @@ export const SelectorLoadObjects = ({
       tableStatus: true,
       accelerationsStatus: true,
     });
-    startLoadingTables(dataSourceName, databaseName);
-    startLoadingAccelerations(dataSourceName);
+    startLoadingTables({ dataSourceName, databaseName });
+    startLoadingAccelerations({ dataSourceName });
   };
 
   useEffect(() => {

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/selectors/source_selector.tsx

@@ -20,13 +20,13 @@ import React, { useEffect, useState } from 'react';
 import { CoreStart } from '../../../../../../../../../../src/core/public';
 import { DATACONNECTIONS_BASE } from '../../../../../../../../common/constants/shared';
 import {
-  CachedDatabase,
   CachedDataSourceStatus,
+  CachedDatabase,
   CreateAccelerationForm,
 } from '../../../../../../../../common/types/data_connections';
 import { CatalogCacheManager } from '../../../../../../../framework/catalog_cache/cache_manager';
 import { useToast } from '../../../../../../common/toast';
-import { hasError, validateDatabase, validateDataTable } from '../create/utils';
+import { hasError, validateDataTable, validateDatabase } from '../create/utils';
 import { SelectorLoadDatabases } from './selector_helpers/load_databases';
 import { SelectorLoadObjects } from './selector_helpers/load_objects';
 
@@ -37,6 +37,7 @@ interface AccelerationDataSourceSelectorProps {
   selectedDatasource: string;
   dataSourcesPreselected: boolean;
   tableFieldsLoading: boolean;
+  dataSourceMDSId?: string;
 }
 
 export const AccelerationDataSourceSelector = ({
@@ -46,6 +47,7 @@ export const AccelerationDataSourceSelector = ({
   selectedDatasource,
   dataSourcesPreselected,
   tableFieldsLoading,
+  dataSourceMDSId,
 }: AccelerationDataSourceSelectorProps) => {
   const { setToast } = useToast();
   const [databases, setDatabases] = useState<Array<EuiComboBoxOptionOption<string>>>([]);
@@ -72,7 +74,7 @@ export const AccelerationDataSourceSelector = ({
   const loadDataSource = () => {
     setLoadingComboBoxes({ ...loadingComboBoxes, dataSource: true });
     http
-      .get(DATACONNECTIONS_BASE)
+      .get(DATACONNECTIONS_BASE + `/dataSourceMDSId=${dataSourceMDSId}`)
       .then((res) => {
         const isValidDataSource = res.some(
           (connection: any) =>

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/query_visual_editor.tsx

@@ -14,12 +14,14 @@ interface QueryVisualEditorProps {
   accelerationFormData: CreateAccelerationForm;
   setAccelerationFormData: React.Dispatch<React.SetStateAction<CreateAccelerationForm>>;
   tableFieldsLoading: boolean;
+  dataSourceMDSId?: string;
 }
 
 export const QueryVisualEditor = ({
   accelerationFormData,
   setAccelerationFormData,
   tableFieldsLoading,
+  dataSourceMDSId,
 }: QueryVisualEditorProps) => {
   return tableFieldsLoading ? (
     <>
@@ -38,6 +40,7 @@ export const QueryVisualEditor = ({
         <SkippingIndexBuilder
           accelerationFormData={accelerationFormData}
           setAccelerationFormData={setAccelerationFormData}
+          dataSourceMDSId={dataSourceMDSId}
         />
       )}
       {accelerationFormData.accelerationIndexType === 'covering' && (

public/components/datasources/components/manage/accelerations/create_accelerations_flyout/visual_editors/skipping_index/generate_fields.tsx

@@ -26,13 +26,15 @@ interface GenerateFieldsProps {
   setAccelerationFormData: React.Dispatch<React.SetStateAction<CreateAccelerationForm>>;
   isSkippingtableLoading: boolean;
   setIsSkippingtableLoading: React.Dispatch<boolean>;
+  dataSourceMDSId?: string;
 }
 
 export const GenerateFields = ({
   accelerationFormData,
   setAccelerationFormData,
   isSkippingtableLoading,
   setIsSkippingtableLoading,
+  dataSourceMDSId,
 }: GenerateFieldsProps) => {
   const [isGenerateRun, setIsGenerateRun] = useState(false);
   const { loadStatus, startLoading, stopLoading: _stopLoading, pollingResult } = useDirectQuery();
@@ -84,7 +86,7 @@ export const GenerateFields = ({
       )}`,
       datasource: accelerationFormData.dataSource,
     };
-    startLoading(requestPayload);
+    startLoading(requestPayload, dataSourceMDSId);
     setIsSkippingtableLoading(true);
     setIsGenerateRun(true);
     setReplaceDefinitionModal(<></>);