Skip to content

Commit 2d18a4f

Browse files
opensearch-trigger-bot[bot]github-actions[bot]
opensearch-trigger-bot[bot]
and
github-actions[bot]
authored
Add datasource field in accelerations cache (opensearch-project#1525) (opensearch-project#1526)
* add datasource in accelerations cache * fixed nits --------- (cherry picked from commit b0f0d9b) Signed-off-by: Shenoy Pratik <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 3fc3b25 commit 2d18a4f

File tree

5 files changed

+298
-65
lines changed

5 files changed

+298
-65
lines changed

‎common/types/data_connections.ts

+11-6
Original file line numberDiff line numberDiff line change
@@ -83,19 +83,19 @@ export interface CachedColumn {
8383

8484
export interface CachedTable {
8585
name: string;
86-
columns: CachedColumn[];
86+
columns?: CachedColumn[];
8787
}
8888

8989
export interface CachedDatabase {
9090
name: string;
9191
tables: CachedTable[];
92-
lastUpdated: string; // Assuming date string in UTC format
92+
lastUpdated: string; // date string in UTC format
9393
status: CachedDataSourceStatus;
9494
}
9595

9696
export interface CachedDataSource {
9797
name: string;
98-
lastUpdated: string; // Assuming date string in UTC format
98+
lastUpdated: string; // date string in UTC format
9999
status: CachedDataSourceStatus;
100100
databases: CachedDatabase[];
101101
}
@@ -115,13 +115,18 @@ export interface CachedAccelerations {
115115
status: string;
116116
}
117117

118-
export interface AccelerationsCacheData {
119-
version: string;
118+
export interface CachedAcclerationByDataSource {
119+
name: string;
120120
accelerations: CachedAccelerations[];
121-
lastUpdated: string; // Assuming date string in UTC format
121+
lastUpdated: string; // date string in UTC format
122122
status: CachedDataSourceStatus;
123123
}
124124

125+
export interface AccelerationsCacheData {
126+
version: string;
127+
dataSources: CachedAcclerationByDataSource[];
128+
}
129+
125130
export interface PollingSuccessResult {
126131
schema: Array<{ name: string; type: string }>;
127132
datarows: Array<Array<string | number | boolean>>;

‎public/framework/catalog_cache/cache_loader.test.tsx

+147-34
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ import {
1111
mockShowTablesPollingResult,
1212
} from '../../../test/datasources';
1313
import {
14+
createLoadQuery,
1415
updateAccelerationsToCache,
1516
updateDatabasesToCache,
1617
updateTablesToCache,
18+
updateToCache,
1719
} from './cache_loader';
1820
import { CatalogCacheManager } from './cache_manager';
1921

@@ -146,10 +148,7 @@ describe('loadCacheTests', () => {
146148
dataSourceName,
147149
expect.objectContaining({
148150
name: databaseName,
149-
tables: [
150-
{ name: 'Table1', columns: [] },
151-
{ name: 'Table2', columns: [] },
152-
],
151+
tables: [{ name: 'Table1' }, { name: 'Table2' }],
153152
lastUpdated: expect.any(String),
154153
status: CachedDataSourceStatus.Updated,
155154
})
@@ -166,55 +165,169 @@ describe('loadCacheTests', () => {
166165
it('should save empty accelerations cache and status failed when polling result is null', () => {
167166
const pollingResult = null;
168167

169-
updateAccelerationsToCache(pollingResult);
168+
updateAccelerationsToCache('sampleDS', pollingResult);
170169

171170
// Verify that saveAccelerationsCache is called with the correct parameters
172171
expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({
173172
version: CATALOG_CACHE_VERSION,
174-
accelerations: [],
175-
lastUpdated: expect.any(String),
176-
status: CachedDataSourceStatus.Failed,
173+
dataSources: [
174+
{
175+
name: 'sampleDS',
176+
accelerations: [],
177+
lastUpdated: expect.any(String),
178+
status: CachedDataSourceStatus.Failed,
179+
},
180+
],
177181
});
178182
});
179183

180184
it('should save new accelerations cache when polling result is not null', () => {
181-
updateAccelerationsToCache(mockShowIndexesPollingResult);
185+
updateAccelerationsToCache('sampleDS', mockShowIndexesPollingResult);
182186

183187
// Verify that saveAccelerationsCache is called with the correct parameters
184188
expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalledWith({
185189
version: CATALOG_CACHE_VERSION,
186-
accelerations: [
187-
{
188-
flintIndexName: 'flint_mys3_default_http_logs_skipping_index',
189-
type: 'skipping',
190-
database: 'default',
191-
table: 'http_logs',
192-
indexName: 'skipping_index',
193-
autoRefresh: false,
194-
status: 'Active',
195-
},
190+
dataSources: [
196191
{
197-
flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index',
198-
type: 'covering',
199-
database: 'default',
200-
table: 'http_logs',
201-
indexName: 'status_clientip_and_day',
202-
autoRefresh: true,
203-
status: 'Active',
192+
name: 'sampleDS',
193+
accelerations: [
194+
{
195+
flintIndexName: 'flint_mys3_default_http_logs_skipping_index',
196+
type: 'skipping',
197+
database: 'default',
198+
table: 'http_logs',
199+
indexName: 'skipping_index',
200+
autoRefresh: false,
201+
status: 'Active',
202+
},
203+
{
204+
flintIndexName: 'flint_mys3_default_http_logs_status_clientip_and_day_index',
205+
type: 'covering',
206+
database: 'default',
207+
table: 'http_logs',
208+
indexName: 'status_clientip_and_day',
209+
autoRefresh: true,
210+
status: 'Active',
211+
},
212+
{
213+
flintIndexName: 'flint_mys3_default_http_count_view',
214+
type: 'materialized',
215+
database: 'default',
216+
table: '',
217+
indexName: 'http_count_view',
218+
autoRefresh: true,
219+
status: 'Active',
220+
},
221+
],
222+
lastUpdated: expect.any(String),
223+
status: CachedDataSourceStatus.Updated,
204224
},
225+
],
226+
});
227+
});
228+
});
229+
230+
describe('updateToCache', () => {
231+
it('should call updateDatabasesToCache when loadCacheType is "databases"', () => {
232+
const loadCacheType = 'databases';
233+
const dataSourceName = 'TestDataSource';
234+
235+
updateToCache(mockShowDatabasesPollingResult, loadCacheType, dataSourceName);
236+
237+
// Verify that addOrUpdateDataSource is called
238+
expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled();
239+
expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled();
240+
expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled();
241+
});
242+
243+
it('should call updateTablesToCache when loadCacheType is "tables"', () => {
244+
const loadCacheType = 'tables';
245+
const dataSourceName = 'TestDataSource';
246+
const databaseName = 'TestDatabase';
247+
248+
CatalogCacheManager.addOrUpdateDataSource({
249+
databases: [
205250
{
206-
flintIndexName: 'flint_mys3_default_http_count_view',
207-
type: 'materialized',
208-
database: 'default',
209-
table: '',
210-
indexName: 'http_count_view',
211-
autoRefresh: true,
212-
status: 'Active',
251+
name: databaseName,
252+
lastUpdated: '',
253+
status: CachedDataSourceStatus.Empty,
254+
tables: [],
213255
},
214256
],
215-
lastUpdated: expect.any(String),
257+
name: dataSourceName,
258+
lastUpdated: new Date().toUTCString(),
216259
status: CachedDataSourceStatus.Updated,
217260
});
261+
262+
updateToCache(mockShowTablesPollingResult, loadCacheType, dataSourceName, databaseName);
263+
264+
// Verify that updateDatabase is called
265+
expect(CatalogCacheManager.addOrUpdateDataSource).toHaveBeenCalled();
266+
expect(CatalogCacheManager.updateDatabase).toHaveBeenCalled();
267+
expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled();
268+
});
269+
270+
it('should call updateAccelerationsToCache when loadCacheType is "accelerations"', () => {
271+
const loadCacheType = 'accelerations';
272+
const dataSourceName = 'TestDataSource';
273+
274+
updateToCache(mockShowIndexesPollingResult, loadCacheType, dataSourceName);
275+
276+
// Verify that saveAccelerationsCache is called
277+
expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled();
278+
expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled();
279+
expect(CatalogCacheManager.saveAccelerationsCache).toHaveBeenCalled();
280+
});
281+
282+
it('should not call any update function when loadCacheType is not recognized', () => {
283+
const pollResults = {};
284+
const loadCacheType = '';
285+
const dataSourceName = 'TestDataSource';
286+
287+
updateToCache(pollResults, loadCacheType, dataSourceName);
288+
289+
// Verify that no update function is called
290+
expect(CatalogCacheManager.addOrUpdateDataSource).not.toHaveBeenCalled();
291+
expect(CatalogCacheManager.updateDatabase).not.toHaveBeenCalled();
292+
expect(CatalogCacheManager.saveAccelerationsCache).not.toHaveBeenCalled();
293+
});
294+
});
295+
296+
describe('createLoadQuery', () => {
297+
it('should create a query for loading databases', () => {
298+
const loadCacheType = 'databases';
299+
const dataSourceName = 'example';
300+
const expectedQuery = 'SHOW SCHEMAS IN `example`';
301+
expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery);
302+
});
303+
304+
it('should create a query for loading tables', () => {
305+
const loadCacheType = 'tables';
306+
const dataSourceName = 'example';
307+
const databaseName = 'test';
308+
const expectedQuery = 'SHOW TABLES IN `example`.`test`';
309+
expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery);
310+
});
311+
312+
it('should create a query for loading accelerations', () => {
313+
const loadCacheType = 'accelerations';
314+
const dataSourceName = 'example';
315+
const expectedQuery = 'SHOW FLINT INDEX in `example`';
316+
expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual(expectedQuery);
317+
});
318+
319+
it('should return an empty string for unknown loadCacheType', () => {
320+
const loadCacheType = 'unknownType';
321+
const dataSourceName = 'example';
322+
expect(createLoadQuery(loadCacheType, dataSourceName)).toEqual('');
323+
});
324+
325+
it('should properly handle backticks in database name', () => {
326+
const loadCacheType = 'tables';
327+
const dataSourceName = 'example';
328+
const databaseName = '`sample`';
329+
const expectedQuery = 'SHOW TABLES IN `example`.`sample`';
330+
expect(createLoadQuery(loadCacheType, dataSourceName, databaseName)).toEqual(expectedQuery);
218331
});
219332
});
220333
});

‎public/framework/catalog_cache/cache_loader.tsx

+12-12
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
*/
55

66
import { useEffect, useState } from 'react';
7-
import {
8-
ASYNC_POLLING_INTERVAL,
9-
CATALOG_CACHE_VERSION,
10-
} from '../../../common/constants/data_sources';
7+
import { ASYNC_POLLING_INTERVAL } from '../../../common/constants/data_sources';
118
import {
129
AsyncPollingResult,
1310
CachedDataSourceStatus,
@@ -80,7 +77,6 @@ export const updateTablesToCache = (
8077
const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows);
8178
const newTables = combinedData.map((row: any) => ({
8279
name: row.tableName,
83-
columns: [],
8480
}));
8581

8682
CatalogCacheManager.updateDatabase(dataSourceName, {
@@ -91,12 +87,15 @@ export const updateTablesToCache = (
9187
});
9288
};
9389

94-
export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) => {
90+
export const updateAccelerationsToCache = (
91+
dataSourceName: string,
92+
pollingResult: AsyncPollingResult
93+
) => {
9594
const currentTime = new Date().toUTCString();
9695

9796
if (!pollingResult) {
98-
CatalogCacheManager.saveAccelerationsCache({
99-
version: CATALOG_CACHE_VERSION,
97+
CatalogCacheManager.addOrUpdateAccelerationsByDataSource({
98+
name: dataSourceName,
10099
accelerations: [],
101100
lastUpdated: currentTime,
102101
status: CachedDataSourceStatus.Failed,
@@ -116,8 +115,8 @@ export const updateAccelerationsToCache = (pollingResult: AsyncPollingResult) =>
116115
status: row.status,
117116
}));
118117

119-
CatalogCacheManager.saveAccelerationsCache({
120-
version: CATALOG_CACHE_VERSION,
118+
CatalogCacheManager.addOrUpdateAccelerationsByDataSource({
119+
name: dataSourceName,
121120
accelerations: newAccelerations,
122121
lastUpdated: currentTime,
123122
status: CachedDataSourceStatus.Updated,
@@ -138,7 +137,7 @@ export const updateToCache = (
138137
updateTablesToCache(dataSourceName, databaseName!, pollResults);
139138
break;
140139
case 'accelerations':
141-
updateAccelerationsToCache(pollResults);
140+
updateAccelerationsToCache(dataSourceName, pollResults);
142141
break;
143142
default:
144143
break;
@@ -189,6 +188,7 @@ export const useLoadToCache = (loadCacheType: LoadCacheType) => {
189188
}, ASYNC_POLLING_INTERVAL);
190189

191190
const startLoading = (dataSourceName: string, databaseName?: string) => {
191+
setLoadStatus(DirectQueryLoadingStatus.SCHEDULED);
192192
setCurrentDataSourceName(dataSourceName);
193193
setCurrentDatabaseName(databaseName);
194194

@@ -272,7 +272,7 @@ export const useLoadTablesToCache = () => {
272272
return { loadStatus, startLoading, stopLoading };
273273
};
274274

275-
export const useAccelerationsToCache = () => {
275+
export const useLoadAccelerationsToCache = () => {
276276
const { loadStatus, startLoading, stopLoading } = useLoadToCache('accelerations');
277277
return { loadStatus, startLoading, stopLoading };
278278
};

0 commit comments

Comments
 (0)