Skip to content

Commit 552d720

Browse files
mengweiericmengweieric
authored
[Explorer] Data selector enhancement and refactoring adoptions (#1759)
* support new changes in data selector Signed-off-by: Eric <[email protected]> * add a missing dependency Signed-off-by: Eric <[email protected]> * changes addressing reviews Signed-off-by: Eric <[email protected]> * fix delete s3 data source issue Signed-off-by: Eric <[email protected]> * error handling Signed-off-by: Eric <[email protected]> * code clean up Signed-off-by: Eric <[email protected]> --------- Signed-off-by: Eric <[email protected]>
1 parent dda72c0 commit 552d720

File tree

7 files changed

+125
-64
lines changed

7 files changed

+125
-64
lines changed

‎common/constants/data_sources.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,7 @@ export const ACCELERATION_AGGREGRATION_FUNCTIONS = [
9090
];
9191

9292
export const SPARK_PARTITION_INFO = `# Partition Information`;
93+
export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector
94+
export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector
95+
export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector
96+
export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector

‎common/constants/shared.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ export const VISUALIZATION_ERROR = {
258258
NO_METRIC: 'Invalid Metric MetaData',
259259
};
260260

261-
export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE';
261+
export const S3_DATA_SOURCE_TYPE = 's3glue';
262262

263263
export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id';
264264
export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache';

‎public/components/event_analytics/explorer/datasources/datasources_selection.tsx

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55

66
import React, { useCallback, useContext, useEffect, useMemo, useState } from 'react';
77
import { batch, useDispatch, useSelector } from 'react-redux';
8+
import { htmlIdGenerator } from '@elastic/eui';
89
import { LogExplorerRouterContext } from '../..';
910
import {
11+
DataSource,
1012
DataSourceGroup,
1113
DataSourceSelectable,
1214
DataSourceType,
@@ -20,6 +22,7 @@ import {
2022
DEFAULT_DATA_SOURCE_TYPE,
2123
DEFAULT_DATA_SOURCE_TYPE_NAME,
2224
INDEX_URL_PARAM_KEY,
25+
OBS_DEFAULT_CLUSTER,
2326
OLLY_QUESTION_URL_PARAM_KEY,
2427
QUERY_LANGUAGE,
2528
} from '../../../../../common/constants/data_sources';
@@ -90,6 +93,21 @@ const removeDataSourceFromURLParams = (currURL: string) => {
9093
}
9194
};
9295

96+
const getMatchedOption = (
97+
dataSourceList: DataSourceGroup[],
98+
dataSourceName: string,
99+
dataSourceType: string
100+
) => {
101+
if (!dataSourceName || !dataSourceType) return [];
102+
for (const dsGroup of dataSourceList) {
103+
const matchedOption = dsGroup.options.find(
104+
(item) => item.type === dataSourceType && item.name === dataSourceName
105+
);
106+
if (matchedOption !== undefined) return [matchedOption];
107+
}
108+
return [];
109+
};
110+
93111
export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
94112
const { dataSources, http } = coreRefs;
95113
const sqlService = new SQLService(http!);
@@ -99,7 +117,11 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
99117
const [activeDataSources, setActiveDataSources] = useState<DataSourceType[]>([]);
100118
const [dataSourceOptionList, setDataSourceOptionList] = useState<DataSourceGroup[]>([]);
101119
const [selectedSources, setSelectedSources] = useState<SelectedDataSource[]>(
102-
getDataSourceState(explorerSearchMetadata.datasources)
120+
getMatchedOption(
121+
dataSourceOptionList,
122+
explorerSearchMetadata.datasources?.[0]?.name || '',
123+
explorerSearchMetadata.datasources?.[0]?.type || ''
124+
)
103125
);
104126

105127
/**
@@ -149,8 +171,14 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
149171
};
150172

151173
useEffect(() => {
152-
setSelectedSources(getDataSourceState(explorerSearchMetadata.datasources));
153-
}, [explorerSearchMetadata.datasources]);
174+
setSelectedSources(
175+
getMatchedOption(
176+
memorizedDataSourceOptionList,
177+
explorerSearchMetadata.datasources?.[0]?.name || '',
178+
explorerSearchMetadata.datasources?.[0]?.type || ''
179+
)
180+
);
181+
}, [explorerSearchMetadata.datasources, dataSourceOptionList]);
154182

155183
const handleDataSetFetchError = useCallback(() => {
156184
return (error: Error) => {
@@ -162,22 +190,33 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
162190
* Subscribe to data source updates and manage the active data sources state.
163191
*/
164192
useEffect(() => {
165-
const subscription = dataSources.dataSourceService.dataSources$.subscribe(
166-
(currentDataSources) => {
193+
const subscription = dataSources.dataSourceService
194+
.getDataSources$()
195+
.subscribe((currentDataSources: DataSource[]) => {
167196
// temporary solution for 2.11 to render OpenSearch / default cluster for observability
168197
// local indices and index patterns, while keep listing all index patterns for data explorer
169198
// it filters the registered index pattern data sources in data plugin, and attach default cluster
170199
// for all indices
171200
setActiveDataSources([
172201
new ObservabilityDefaultDataSource({
202+
id: htmlIdGenerator(OBS_DEFAULT_CLUSTER)(DEFAULT_DATA_SOURCE_TYPE),
173203
name: DEFAULT_DATA_SOURCE_NAME,
174204
type: DEFAULT_DATA_SOURCE_TYPE,
175-
metadata: null,
205+
metadata: {
206+
ui: {
207+
label: DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME,
208+
groupType: DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME,
209+
selector: {
210+
displayDatasetsAsSource: false, // when true, selector UI will render data sets with source by calling getDataSets()
211+
},
212+
},
213+
},
176214
}),
177-
...Object.values(currentDataSources).filter((ds) => ds.type !== DEFAULT_DATA_SOURCE_TYPE),
215+
...Object.values(currentDataSources).filter(
216+
(ds) => ds.getType() !== DEFAULT_DATA_SOURCE_TYPE
217+
),
178218
]);
179-
}
180-
);
219+
});
181220

182221
return () => subscription.unsubscribe();
183222
}, []);
@@ -259,6 +298,10 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
259298
});
260299
}, [dataSourceOptionList]);
261300

301+
const onRefresh = useCallback(() => {
302+
dataSources.dataSourceService.reload();
303+
}, [dataSources.dataSourceService]);
304+
262305
return (
263306
<DataSourceSelectable
264307
className="dsc-selector"
@@ -267,9 +310,10 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
267310
setDataSourceOptionList={setDataSourceOptionList}
268311
selectedSources={selectedSources}
269312
onDataSourceSelect={handleSourceChange}
270-
onFetchDataSetError={handleDataSetFetchError}
271313
singleSelection={{ asPlainText: true }}
272314
dataSourceSelectorConfigs={DATA_SOURCE_SELECTOR_CONFIGS}
315+
onGetDataSetError={handleDataSetFetchError}
316+
onRefresh={onRefresh}
273317
/>
274318
);
275319
};

‎public/components/event_analytics/explorer/log_explorer.tsx

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
/* eslint-disable react-hooks/exhaustive-deps */
66
import { isEmpty } from 'lodash';
77
import React, { useContext, useEffect, useRef, useState } from 'react';
8-
import { batch, useSelector, useDispatch } from 'react-redux';
8+
import { useSelector } from 'react-redux';
99
import { useHistory } from 'react-router-dom';
1010
import { LogExplorerRouterContext } from '..';
1111
import {
@@ -26,14 +26,6 @@ import {
2626
} from '../../../../common/constants/shared';
2727
import { coreRefs } from '../../../../public/framework/core_refs';
2828

29-
import { init as initFields } from '../../event_analytics/redux/slices/field_slice';
30-
import { init as initPatterns } from '../../event_analytics/redux/slices/patterns_slice';
31-
import { init as initQueryResult } from '../../event_analytics/redux/slices/query_result_slice';
32-
import { init as initQuery } from '../../event_analytics/redux/slices/query_slice';
33-
import { init as initVisualizationConfig } from '../../event_analytics/redux/slices/viualization_config_slice';
34-
import { resetSummary as initQueryAssistSummary } from '../../event_analytics/redux/slices/query_assistant_summarization_slice';
35-
import { init as initSearchMetaData } from '../../event_analytics/redux/slices/search_meta_data_slice';
36-
3729
const searchBarConfigs = {
3830
[TAB_EVENT_ID]: {
3931
showSaveButton: true,
@@ -60,7 +52,6 @@ export const LogExplorer = ({
6052
dataSourcePluggables,
6153
}: ILogExplorerProps) => {
6254
const history = useHistory();
63-
const dispatch = useDispatch();
6455
const routerContext = useContext(LogExplorerRouterContext);
6556
const tabIds = useSelector(selectQueryTabs).queryTabIds.filter(
6657
(tabid: string) => !tabid.match(APP_ANALYTICS_TAB_ID_REGEX)
@@ -97,20 +88,8 @@ export const LogExplorer = ({
9788
useEffect(() => {
9889
if (!isEmpty(savedObjectId)) {
9990
dispatchSavedObjectId();
100-
} else {
101-
// below piece of code was added to simulate creating a new tab if saved obj isn't being loaded,
102-
// since tabs being visually removed means 'new tab' cannot be created any other way
103-
const tabId = tabIds[0];
104-
batch(() => {
105-
dispatch(initQuery({ tabId }));
106-
dispatch(initQueryResult({ tabId }));
107-
dispatch(initFields({ tabId }));
108-
dispatch(initVisualizationConfig({ tabId }));
109-
dispatch(initPatterns({ tabId }));
110-
dispatch(initQueryAssistSummary({ tabId }));
111-
dispatch(initSearchMetaData({ tabId }));
112-
});
11391
}
92+
11493
if (routerContext && routerContext.searchParams.has(CREATE_TAB_PARAM_KEY)) {
11594
// need to wait for current redux event loop to finish
11695
setImmediate(() => {

‎public/framework/datasources/obs_opensearch_datasource.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,26 @@
66
import { DataSource } from '../../../../../src/plugins/data/public';
77

88
interface DataSourceConfig {
9+
id: string;
910
name: string;
1011
type: string;
1112
metadata: any;
1213
}
1314

1415
export class ObservabilityDefaultDataSource extends DataSource<any, any, any, any, any> {
15-
constructor({ name, type, metadata }: DataSourceConfig) {
16-
super(name, type, metadata);
16+
constructor({ id, name, type, metadata }: DataSourceConfig) {
17+
super({ id, name, type, metadata });
1718
}
1819

19-
async getDataSet(dataSetParams?: any) {
20+
async getDataSet() {
2021
return ['Default data source'];
2122
}
2223

2324
async testConnection(): Promise<boolean> {
2425
return true;
2526
}
2627

27-
async runQuery(queryParams: any) {
28+
async runQuery() {
2829
return null;
2930
}
3031
}

‎public/framework/datasources/s3_datasource.ts

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,23 @@ interface DataSourceConfig {
99
name: string;
1010
type: string;
1111
metadata: any;
12+
id: string;
1213
}
1314

14-
export class S3DataSource extends DataSource<any, any, any, any, any> {
15-
constructor({ name, type, metadata }: DataSourceConfig) {
16-
super(name, type, metadata);
15+
export class S3DataSource extends DataSource {
16+
constructor({ id, name, type, metadata }: DataSourceConfig) {
17+
super({ id, name, type, metadata });
1718
}
1819

19-
async getDataSet(dataSetParams?: any) {
20-
return [this.getName()];
20+
async getDataSet() {
21+
return { dataSets: [this.getName()] };
2122
}
2223

23-
async testConnection(): Promise<void> {
24-
throw new Error('This operation is not supported for this class.');
24+
async testConnection(): Promise<boolean> {
25+
return true;
2526
}
2627

27-
async runQuery(queryParams: any) {
28-
return null;
28+
async runQuery() {
29+
return { data: {} };
2930
}
3031
}

‎public/plugin.tsx

Lines changed: 49 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
import { i18n } from '@osd/i18n';
76
import React from 'react';
7+
import { i18n } from '@osd/i18n';
8+
import { htmlIdGenerator } from '@elastic/eui';
89
import {
910
AppCategory,
1011
AppMountParameters,
@@ -46,7 +47,7 @@ import {
4647
observabilityTracesID,
4748
observabilityTracesPluginOrder,
4849
observabilityTracesTitle,
49-
S3_DATASOURCE_TYPE,
50+
S3_DATA_SOURCE_TYPE,
5051
} from '../common/constants/shared';
5152
import { QueryManager } from '../common/query_manager';
5253
import { AssociatedObject, CachedAcceleration } from '../common/types/data_connections';
@@ -95,6 +96,12 @@ import {
9596
ObservabilityStart,
9697
SetupDependencies,
9798
} from './types';
99+
import {
100+
DATA_SOURCE_TYPES,
101+
OBS_S3_DATA_SOURCE,
102+
S3_DATA_SOURCE_GROUP_DISPLAY_NAME,
103+
S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME,
104+
} from '../common/constants/data_sources';
98105

99106
interface PublicConfig {
100107
query_assist: {
@@ -387,38 +394,63 @@ export class ObservabilityPlugin
387394
coreRefs.overlays = core.overlays;
388395

389396
const { dataSourceService, dataSourceFactory } = startDeps.data.dataSources;
397+
dataSourceFactory.registerDataSourceType(S3_DATA_SOURCE_TYPE, S3DataSource);
398+
399+
const getDataSourceTypeLabel = (type: string) => {
400+
if (type === DATA_SOURCE_TYPES.S3Glue) return S3_DATA_SOURCE_GROUP_DISPLAY_NAME;
401+
if (type === DATA_SOURCE_TYPES.SPARK) return S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME;
402+
return `${type.charAt(0).toUpperCase()}${type.slice(1)}`;
403+
};
390404

391405
// register all s3 datasources
392-
const registerS3Datasource = () => {
393-
dataSourceFactory.registerDataSourceType(S3_DATASOURCE_TYPE, S3DataSource);
394-
core.http.get(`${DATACONNECTIONS_BASE}`).then((s3DataSources) => {
395-
s3DataSources.map((s3ds) => {
396-
dataSourceService.registerDataSource(
397-
dataSourceFactory.getDataSourceInstance(S3_DATASOURCE_TYPE, {
398-
name: s3ds.name,
399-
type: s3ds.connector.toLowerCase(),
400-
metadata: s3ds,
401-
})
402-
);
406+
const registerDataSources = () => {
407+
try {
408+
core.http.get(`${DATACONNECTIONS_BASE}`).then((s3DataSources) => {
409+
s3DataSources.map((s3ds) => {
410+
dataSourceService.registerDataSource(
411+
dataSourceFactory.getDataSourceInstance(S3_DATA_SOURCE_TYPE, {
412+
id: htmlIdGenerator(OBS_S3_DATA_SOURCE)(),
413+
name: s3ds.name,
414+
type: s3ds.connector.toLowerCase(),
415+
metadata: {
416+
...s3ds.properties,
417+
ui: {
418+
label: s3ds.name,
419+
typeLabel: getDataSourceTypeLabel(s3ds.connector.toLowerCase()),
420+
groupType: s3ds.connector.toLowerCase(),
421+
selector: {
422+
displayDatasetsAsSource: false,
423+
},
424+
},
425+
},
426+
})
427+
);
428+
});
403429
});
404-
});
430+
} catch (error) {
431+
console.error('Error registering S3 datasources', error);
432+
}
405433
};
406434

435+
dataSourceService.registerDataSourceFetchers([
436+
{ type: S3_DATA_SOURCE_TYPE, registerDataSources },
437+
]);
438+
407439
if (startDeps.securityDashboards) {
408440
core.http
409441
.get(SECURITY_PLUGIN_ACCOUNT_API)
410442
.then(() => {
411-
registerS3Datasource();
443+
registerDataSources();
412444
})
413445
.catch((e) => {
414446
if (e?.response?.status !== 401) {
415447
// accounts api should not return any error status other than 401 if security installed,
416448
// this datasource register is included just in case
417-
registerS3Datasource();
449+
registerDataSources();
418450
}
419451
});
420452
} else {
421-
registerS3Datasource();
453+
registerDataSources();
422454
}
423455

424456
core.http.intercept({

0 commit comments

Comments
 (0)