Skip to content

Commit 1c5ad6c

Browse files
authored
[MDS] Support Vega Visualizations (#5975)
* Add MDS support for Vega Signed-off-by: Huy Nguyen <[email protected]> * Refactor field to data_source_id Signed-off-by: Huy Nguyen <[email protected]> * Add to CHANGELOG.md Signed-off-by: Huy Nguyen <[email protected]> * Added test cases and renamed field to use data_source_name Signed-off-by: Huy Nguyen <[email protected]> * Add prefix datasource name test case and add example in default hjson Signed-off-by: Huy Nguyen <[email protected]> * Move CHANGELOG to appropriate section Signed-off-by: Huy Nguyen <[email protected]> * Increased test coverage of search() method Signed-off-by: Huy Nguyen <[email protected]> --------- Signed-off-by: Huy Nguyen <[email protected]>
1 parent 6f4d814 commit 1c5ad6c

10 files changed

+271
-27
lines changed

CHANGELOG.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
1111
### 🛡 Security
1212

1313
### 📈 Features/Enhancements
14-
- [MD]Change cluster selector component name to data source selector ([#6042](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6042))
14+
- [MD]Change cluster selector component name to data source selector ([#6042](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6042))
1515
- [Multiple Datasource] Add interfaces to register add-on authentication method from plug-in module ([#5851](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5851))
1616
- [Multiple Datasource] Able to Hide "Local Cluster" option from datasource DropDown ([#5827](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5827))
1717
- [Multiple Datasource] Add api registry and allow it to be added into client config in data source plugin ([#5895](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5895))
@@ -26,6 +26,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
2626
- [Multiple Datasource] Expose a few properties for customize the appearance of the data source selector component ([#6057](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6057))
2727
- [Multiple Datasource] Create data source menu component able to be mount to nav bar ([#6082](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6082))
2828
- [Multiple Datasource] Handle form values(request payload) if the selected type is available in the authentication registry ([#6049](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6049))
29+
- [Multiple Datasource] Add Vega support to MDS by specifying a data source name in the Vega spec ([#5975](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5975))
2930

3031
### 🐛 Bug Fixes
3132

@@ -39,7 +40,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
3940
- [osd/std] Add additional recovery from false-positives in handling of long numerals ([#5956](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5956))
4041
- [BUG][Multiple Datasource] Fix missing customApiRegistryPromise param for test connection ([#5944](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5944))
4142
- [BUG][Multiple Datasource] Add a migration function for datasource to add migrationVersion field ([#6025](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6025))
42-
- [BUG][MD]Expose picker using function in data source management plugin setup([#6030](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6030))
43+
- [BUG][MD]Expose picker using function in data source management plugin setup([#6030](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6030))
4344

4445
### 🚞 Infrastructure
4546

src/plugins/vis_type_vega/opensearch_dashboards.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"inspector",
1212
"uiActions"
1313
],
14-
"optionalPlugins": ["home", "usageCollection"],
14+
"optionalPlugins": ["home", "usageCollection", "dataSource"],
1515
"requiredBundles": [
1616
"opensearchDashboardsUtils",
1717
"opensearchDashboardsReact",

src/plugins/vis_type_vega/public/data_model/opensearch_query_parser.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ export class OpenSearchQueryParser {
229229
name: getRequestName(r, index),
230230
}));
231231

232-
const data$ = this._searchAPI.search(opensearchSearches);
232+
const data$ = await this._searchAPI.search(opensearchSearches);
233233

234234
const results = await data$.toPromise();
235235

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { SavedObjectsClientContract, SavedObjectsFindOptions } from 'opensearch-dashboards/public';
7+
import { SearchAPI, SearchAPIDependencies } from './search_api';
8+
import { ISearchStart } from 'src/plugins/data/public';
9+
import { IUiSettingsClient } from 'opensearch-dashboards/public';
10+
11+
jest.mock('rxjs', () => ({
12+
combineLatest: jest.fn().mockImplementation((obj) => obj),
13+
}));
14+
15+
jest.mock('../../../data/public', () => ({
16+
getSearchParamsFromRequest: jest.fn().mockImplementation((obj, _) => obj),
17+
}));
18+
19+
interface MockSearch {
20+
params?: Record<string, unknown>;
21+
dataSourceId?: string;
22+
pipe: () => {};
23+
}
24+
25+
describe('SearchAPI.search', () => {
26+
// This will only test that searchApiParams were correctly set. As such, every other function can be mocked
27+
const getSearchAPI = (dataSourceEnabled: boolean) => {
28+
const savedObjectsClient = {} as SavedObjectsClientContract;
29+
30+
const searchStartMock = {} as ISearchStart;
31+
searchStartMock.search = jest.fn().mockImplementation((obj, _) => {
32+
const mockedSearchResults = {} as MockSearch;
33+
mockedSearchResults.params = obj;
34+
mockedSearchResults.pipe = jest.fn().mockReturnValue(mockedSearchResults.params);
35+
return mockedSearchResults;
36+
});
37+
38+
const uiSettings = {} as IUiSettingsClient;
39+
uiSettings.get = jest.fn().mockReturnValue(0);
40+
uiSettings.get.bind = jest.fn().mockReturnValue(0);
41+
42+
const dependencies = {
43+
savedObjectsClient,
44+
dataSourceEnabled,
45+
search: searchStartMock,
46+
uiSettings,
47+
} as SearchAPIDependencies;
48+
const searchAPI = new SearchAPI(dependencies);
49+
searchAPI.findDataSourceIdbyName = jest.fn().mockImplementation((name) => {
50+
if (!dataSourceEnabled) {
51+
throw new Error();
52+
}
53+
if (name === 'exampleName') {
54+
return Promise.resolve('some-id');
55+
}
56+
});
57+
58+
return searchAPI;
59+
};
60+
61+
test('If MDS is disabled and there is no datasource, return params without datasource id', async () => {
62+
const searchAPI = getSearchAPI(false);
63+
const requests = [{ name: 'example-id' }];
64+
const fetchParams = ((await searchAPI.search(requests)) as unknown) as MockSearch[];
65+
expect(fetchParams[0].params).toBe(requests[0]);
66+
expect(fetchParams[0].hasOwnProperty('dataSourceId')).toBe(false);
67+
});
68+
69+
test('If MDS is disabled and there is a datasource, it should throw an errorr', () => {
70+
const searchAPI = getSearchAPI(false);
71+
const requests = [{ name: 'example-id', data_source_name: 'non-existent-datasource' }];
72+
expect(searchAPI.search(requests)).rejects.toThrowError();
73+
});
74+
75+
test('If MDS is enabled and there is no datasource, return params without datasource id', async () => {
76+
const searchAPI = getSearchAPI(true);
77+
const requests = [{ name: 'example-id' }];
78+
const fetchParams = ((await searchAPI.search(requests)) as unknown) as MockSearch[];
79+
expect(fetchParams[0].params).toBe(requests[0]);
80+
expect(fetchParams[0].hasOwnProperty('dataSourceId')).toBe(false);
81+
});
82+
83+
test('If MDS is enabled and there is a datasource, return params with datasource id', async () => {
84+
const searchAPI = getSearchAPI(true);
85+
const requests = [{ name: 'example-id', data_source_name: 'exampleName' }];
86+
const fetchParams = ((await searchAPI.search(requests)) as unknown) as MockSearch[];
87+
expect(fetchParams[0].hasOwnProperty('params')).toBe(true);
88+
expect(fetchParams[0].dataSourceId).toBe('some-id');
89+
});
90+
});
91+
92+
describe('SearchAPI.findDataSourceIdbyName', () => {
93+
const savedObjectsClient = {} as SavedObjectsClientContract;
94+
savedObjectsClient.find = jest.fn().mockImplementation((query: SavedObjectsFindOptions) => {
95+
if (query.search === `"uniqueDataSource"`) {
96+
return Promise.resolve({
97+
total: 1,
98+
savedObjects: [{ id: 'some-datasource-id', attributes: { title: 'uniqueDataSource' } }],
99+
});
100+
} else if (query.search === `"duplicateDataSource"`) {
101+
return Promise.resolve({
102+
total: 2,
103+
savedObjects: [
104+
{ id: 'some-datasource-id', attributes: { title: 'duplicateDataSource' } },
105+
{ id: 'some-other-datasource-id', attributes: { title: 'duplicateDataSource' } },
106+
],
107+
});
108+
} else if (query.search === `"DataSource"`) {
109+
return Promise.resolve({
110+
total: 2,
111+
savedObjects: [
112+
{ id: 'some-datasource-id', attributes: { title: 'DataSource' } },
113+
{ id: 'some-other-datasource-id', attributes: { title: 'DataSource Copy' } },
114+
],
115+
});
116+
} else {
117+
return Promise.resolve({
118+
total: 0,
119+
savedObjects: [],
120+
});
121+
}
122+
});
123+
124+
const getSearchAPI = (dataSourceEnabled: boolean) => {
125+
const dependencies = { savedObjectsClient, dataSourceEnabled } as SearchAPIDependencies;
126+
return new SearchAPI(dependencies);
127+
};
128+
129+
test('If dataSource is disabled, throw error', () => {
130+
const searchAPI = getSearchAPI(false);
131+
expect(searchAPI.findDataSourceIdbyName('nonexistentDataSource')).rejects.toThrowError(
132+
'data_source_name cannot be used because data_source.enabled is false'
133+
);
134+
});
135+
136+
test('If dataSource is enabled but no matching dataSourceName, then throw error', () => {
137+
const searchAPI = getSearchAPI(true);
138+
expect(searchAPI.findDataSourceIdbyName('nonexistentDataSource')).rejects.toThrowError(
139+
'Expected exactly 1 result for data_source_name "nonexistentDataSource" but got 0 results'
140+
);
141+
});
142+
143+
test('If dataSource is enabled but multiple dataSourceNames, then throw error', () => {
144+
const searchAPI = getSearchAPI(true);
145+
expect(searchAPI.findDataSourceIdbyName('duplicateDataSource')).rejects.toThrowError(
146+
'Expected exactly 1 result for data_source_name "duplicateDataSource" but got 2 results'
147+
);
148+
});
149+
150+
test('If dataSource is enabled but only one dataSourceName, then return id', async () => {
151+
const searchAPI = getSearchAPI(true);
152+
expect(await searchAPI.findDataSourceIdbyName('uniqueDataSource')).toBe('some-datasource-id');
153+
});
154+
155+
test('If dataSource is enabled and the dataSourceName is a prefix of another, ensure the prefix is only returned', async () => {
156+
const searchAPI = getSearchAPI(true);
157+
expect(await searchAPI.findDataSourceIdbyName('DataSource')).toBe('some-datasource-id');
158+
});
159+
});

src/plugins/vis_type_vega/public/data_model/search_api.ts

+68-20
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import { combineLatest } from 'rxjs';
3232
import { map, tap } from 'rxjs/operators';
3333
import { CoreStart, IUiSettingsClient } from 'opensearch-dashboards/public';
34+
import { SavedObjectsClientContract } from 'src/core/public';
35+
import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
3436
import {
3537
getSearchParamsFromRequest,
3638
SearchRequest,
@@ -45,6 +47,8 @@ export interface SearchAPIDependencies {
4547
uiSettings: IUiSettingsClient;
4648
injectedMetadata: CoreStart['injectedMetadata'];
4749
search: DataPublicPluginStart['search'];
50+
dataSourceEnabled: boolean;
51+
savedObjectsClient: SavedObjectsClientContract;
4852
}
4953

5054
export class SearchAPI {
@@ -54,31 +58,75 @@ export class SearchAPI {
5458
public readonly inspectorAdapters?: VegaInspectorAdapters
5559
) {}
5660

57-
search(searchRequests: SearchRequest[]) {
61+
async search(searchRequests: SearchRequest[]) {
5862
const { search } = this.dependencies.search;
5963
const requestResponders: any = {};
6064

6165
return combineLatest(
62-
searchRequests.map((request) => {
63-
const requestId = request.name;
64-
const params = getSearchParamsFromRequest(request, {
65-
getConfig: this.dependencies.uiSettings.get.bind(this.dependencies.uiSettings),
66-
});
67-
68-
if (this.inspectorAdapters) {
69-
requestResponders[requestId] = this.inspectorAdapters.requests.start(requestId, request);
70-
requestResponders[requestId].json(params.body);
71-
}
72-
73-
return search({ params }, { abortSignal: this.abortSignal }).pipe(
74-
tap((data) => this.inspectSearchResult(data, requestResponders[requestId])),
75-
map((data) => ({
76-
name: requestId,
77-
rawResponse: data.rawResponse,
78-
}))
79-
);
80-
})
66+
await Promise.all(
67+
searchRequests.map(async (request) => {
68+
const requestId = request.name;
69+
const dataSourceId = !!request.data_source_name
70+
? await this.findDataSourceIdbyName(request.data_source_name)
71+
: undefined;
72+
73+
const params = getSearchParamsFromRequest(request, {
74+
getConfig: this.dependencies.uiSettings.get.bind(this.dependencies.uiSettings),
75+
});
76+
77+
if (this.inspectorAdapters) {
78+
requestResponders[requestId] = this.inspectorAdapters.requests.start(
79+
requestId,
80+
request
81+
);
82+
requestResponders[requestId].json(params.body);
83+
}
84+
85+
const searchApiParams =
86+
dataSourceId && this.dependencies.dataSourceEnabled
87+
? { params, dataSourceId }
88+
: { params };
89+
90+
return search(searchApiParams, { abortSignal: this.abortSignal }).pipe(
91+
tap((data) => this.inspectSearchResult(data, requestResponders[requestId])),
92+
map((data) => ({
93+
name: requestId,
94+
rawResponse: data.rawResponse,
95+
}))
96+
);
97+
})
98+
)
99+
);
100+
}
101+
102+
async findDataSourceIdbyName(dataSourceName: string) {
103+
if (!this.dependencies.dataSourceEnabled) {
104+
throw new Error('data_source_name cannot be used because data_source.enabled is false');
105+
}
106+
const dataSources = await this.dataSourceFindQuery(dataSourceName);
107+
108+
// In the case that data_source_name is a prefix of another name, match exact data_source_name
109+
const possibleDataSourceIds = dataSources.savedObjects.filter(
110+
(obj) => obj.attributes.title === dataSourceName
81111
);
112+
113+
if (possibleDataSourceIds.length !== 1) {
114+
throw new Error(
115+
`Expected exactly 1 result for data_source_name "${dataSourceName}" but got ${possibleDataSourceIds.length} results`
116+
);
117+
}
118+
119+
return possibleDataSourceIds.pop()?.id;
120+
}
121+
122+
async dataSourceFindQuery(dataSourceName: string) {
123+
return await this.dependencies.savedObjectsClient.find<DataSourceAttributes>({
124+
type: 'data-source',
125+
perPage: 10,
126+
search: `"${dataSourceName}"`,
127+
searchFields: ['title'],
128+
fields: ['id', 'title'],
129+
});
82130
}
83131

84132
public resetSearchStats() {

src/plugins/vis_type_vega/public/data_model/types.ts

+1
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ export interface UrlObject {
186186
[CONSTANTS.TYPE]?: string;
187187
name?: string;
188188
index?: string;
189+
data_source_name?: string;
189190
body?: Body;
190191
size?: number;
191192
timeout?: string;

src/plugins/vis_type_vega/public/default.spec.hjson

+4
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@
2929

3030
// Which index to search
3131
index: _all
32+
33+
// If "data_source.enabled: true", optionally set the data source name to query from (omit field if querying from local cluster)
34+
// data_source_name: Example US Cluster
35+
3236
// Aggregate data by the time field into time buckets, counting the number of documents in each bucket.
3337
body: {
3438
aggs: {

src/plugins/vis_type_vega/public/plugin.ts

+14-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
* under the License.
2929
*/
3030

31+
import { DataSourcePluginSetup } from 'src/plugins/data_source/public';
3132
import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from '../../../core/public';
3233
import { Plugin as ExpressionsPublicPlugin } from '../../expressions/public';
3334
import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public';
@@ -41,6 +42,8 @@ import {
4142
setUISettings,
4243
setMapsLegacyConfig,
4344
setInjectedMetadata,
45+
setDataSourceEnabled,
46+
setSavedObjectsClient,
4447
} from './services';
4548

4649
import { createVegaFn } from './expressions/vega_fn';
@@ -69,6 +72,7 @@ export interface VegaPluginSetupDependencies {
6972
visualizations: VisualizationsSetup;
7073
inspector: InspectorSetup;
7174
data: DataPublicPluginSetup;
75+
dataSource?: DataSourcePluginSetup;
7276
mapsLegacy: any;
7377
}
7478

@@ -88,14 +92,22 @@ export class VegaPlugin implements Plugin<Promise<void>, void> {
8892

8993
public async setup(
9094
core: CoreSetup,
91-
{ inspector, data, expressions, visualizations, mapsLegacy }: VegaPluginSetupDependencies
95+
{
96+
inspector,
97+
data,
98+
expressions,
99+
visualizations,
100+
mapsLegacy,
101+
dataSource,
102+
}: VegaPluginSetupDependencies
92103
) {
93104
setInjectedVars({
94105
enableExternalUrls: this.initializerContext.config.get().enableExternalUrls,
95106
emsTileLayerId: core.injectedMetadata.getInjectedVar('emsTileLayerId', true),
96107
});
97108
setUISettings(core.uiSettings);
98109
setMapsLegacyConfig(mapsLegacy.config);
110+
setDataSourceEnabled({ enabled: !!dataSource });
99111

100112
const visualizationDependencies: Readonly<VegaVisualizationDependencies> = {
101113
core,
@@ -116,6 +128,7 @@ export class VegaPlugin implements Plugin<Promise<void>, void> {
116128
public start(core: CoreStart, { data, uiActions }: VegaPluginStartDependencies) {
117129
setNotifications(core.notifications);
118130
setData(data);
131+
setSavedObjectsClient(core.savedObjects);
119132
setUiActions(uiActions);
120133
setInjectedMetadata(core.injectedMetadata);
121134
}

0 commit comments

Comments
 (0)