Skip to content

Commit 8874c8c

Browse files
SwiddisSwiddis
authored
Add conditional installation for S3 integrations (#1518)
* Add workflows to integration format Signed-off-by: Simeon Widdis <[email protected]> * Render integration workflows on frontend Signed-off-by: Simeon Widdis <[email protected]> * Add ability to toggle workflows to frontend Signed-off-by: Simeon Widdis <[email protected]> * Add workflows to integration build options Signed-off-by: Simeon Widdis <[email protected]> * Add asset workflow filtering to builder Signed-off-by: Simeon Widdis <[email protected]> * Add enabled workflows to setup request Signed-off-by: Simeon Widdis <[email protected]> * Don't allow integration setup if no workflows enabled Signed-off-by: Simeon Widdis <[email protected]> * Add workflows to other integrations Signed-off-by: Simeon Widdis <[email protected]> * Improve header for workflows section Signed-off-by: Simeon Widdis <[email protected]> * Update snapshots Signed-off-by: Simeon Widdis <[email protected]> --------- Signed-off-by: Simeon Widdis <[email protected]>
1 parent b0f0d9b commit 8874c8c

File tree

16 files changed

+1579
-51
lines changed

16 files changed

+1579
-51
lines changed

‎public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap

+1,247
Large diffs are not rendered by default.

‎public/components/integrations/components/__tests__/setup_integration.test.tsx

+15
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,19 @@ describe('Integration Setup Page', () => {
5353
expect(wrapper).toMatchSnapshot();
5454
});
5555
});
56+
57+
it('Renders the S3 connector form without workflows', async () => {
58+
const wrapper = mount(
59+
<SetupIntegrationForm
60+
config={{ ...TEST_INTEGRATION_SETUP_INPUTS, connectionType: 's3' }}
61+
updateConfig={() => {}}
62+
integration={{ ...TEST_INTEGRATION_CONFIG, workflows: undefined }}
63+
setupCallout={{ show: false }}
64+
/>
65+
);
66+
67+
await waitFor(() => {
68+
expect(wrapper).toMatchSnapshot();
69+
});
70+
});
5671
});

‎public/components/integrations/components/create_integration_helpers.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,8 @@ export async function addIntegrationRequest(
282282
integration: IntegrationConfig,
283283
setToast: (title: string, color?: Color, text?: string | undefined) => void,
284284
name?: string,
285-
dataSource?: string
285+
dataSource?: string,
286+
workflows?: string[]
286287
): Promise<boolean> {
287288
const http = coreRefs.http!;
288289
if (addSample) {
@@ -298,7 +299,7 @@ export async function addIntegrationRequest(
298299

299300
let response: boolean = await http
300301
.post(`${INTEGRATIONS_BASE}/store/${templateName}`, {
301-
body: JSON.stringify({ name, dataSource }),
302+
body: JSON.stringify({ name, dataSource, workflows }),
302303
})
303304
.then((res) => {
304305
setToast(`${name} integration successfully added!`, 'success');

‎public/components/integrations/components/setup_integration.tsx

+105-15
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
EuiButton,
99
EuiButtonEmpty,
1010
EuiCallOut,
11+
EuiCheckableCard,
1112
EuiComboBox,
1213
EuiEmptyPrompt,
1314
EuiFieldText,
@@ -42,6 +43,7 @@ export interface IntegrationSetupInputs {
4243
connectionLocation: string;
4344
checkpointLocation: string;
4445
connectionTableName: string;
46+
enabledWorkflows: string[];
4547
}
4648

4749
type SetupCallout = { show: true; title: string; color?: Color; text?: string } | { show: false };
@@ -182,6 +184,38 @@ const runQuery = async (
182184
}
183185
};
184186

187+
export function SetupWorkflowSelector({
188+
integration,
189+
useWorkflows,
190+
toggleWorkflow,
191+
}: {
192+
integration: IntegrationConfig;
193+
useWorkflows: Map<string, boolean>;
194+
toggleWorkflow: (name: string) => void;
195+
}) {
196+
if (!integration.workflows) {
197+
return null;
198+
}
199+
200+
const cards = integration.workflows.map((workflow) => {
201+
return (
202+
<EuiCheckableCard
203+
id={`workflow-checkbox-${workflow.name}`}
204+
key={workflow.name}
205+
label={workflow.label}
206+
checkableType="checkbox"
207+
value={workflow.name}
208+
checked={useWorkflows.get(workflow.name)}
209+
onChange={() => toggleWorkflow(workflow.name)}
210+
>
211+
{workflow.description}
212+
</EuiCheckableCard>
213+
);
214+
});
215+
216+
return cards;
217+
}
218+
185219
export function SetupIntegrationForm({
186220
config,
187221
updateConfig,
@@ -197,6 +231,25 @@ export function SetupIntegrationForm({
197231
const [isBucketBlurred, setIsBucketBlurred] = useState(false);
198232
const [isCheckpointBlurred, setIsCheckpointBlurred] = useState(false);
199233

234+
const [useWorkflows, setUseWorkflows] = useState(new Map<string, boolean>());
235+
const toggleWorkflow = (name: string) => {
236+
setUseWorkflows(new Map(useWorkflows.set(name, !useWorkflows.get(name))));
237+
};
238+
239+
useEffect(() => {
240+
if (integration.workflows) {
241+
setUseWorkflows(new Map(integration.workflows.map((w) => [w.name, w.enabled_by_default])));
242+
}
243+
}, [integration.workflows]);
244+
245+
useEffect(() => {
246+
updateConfig({
247+
enabledWorkflows: [...useWorkflows.entries()].filter((w) => w[1]).map((w) => w[0]),
248+
});
249+
// If we add the updateConfig dep here, rendering crashes with "Maximum update depth exceeded"
250+
// eslint-disable-next-line react-hooks/exhaustive-deps
251+
}, [useWorkflows]);
252+
200253
useEffect(() => {
201254
const updateDataSources = async () => {
202255
const data = await suggestDataSources(config.connectionType);
@@ -339,12 +392,47 @@ export function SetupIntegrationForm({
339392
}}
340393
/>
341394
</EuiFormRow>
395+
{integration.workflows ? (
396+
<>
397+
<EuiSpacer />
398+
<EuiText>
399+
<h3>Installation Flows</h3>
400+
</EuiText>
401+
<EuiSpacer />
402+
<EuiFormRow
403+
label={'Flows'}
404+
helpText={
405+
'Select from the available asset types based on your use case. Choose at least one.'
406+
}
407+
isInvalid={![...useWorkflows.values()].includes(true)}
408+
error={['Must select at least one workflow.']}
409+
>
410+
<SetupWorkflowSelector
411+
integration={integration}
412+
useWorkflows={useWorkflows}
413+
toggleWorkflow={toggleWorkflow}
414+
/>
415+
</EuiFormRow>
416+
</>
417+
) : null}
342418
</>
343419
) : null}
344420
</EuiForm>
345421
);
346422
}
347423

424+
const prepareQuery = (query: string, config: IntegrationSetupInputs): string => {
425+
let queryStr = query.replaceAll(
426+
'{table_name}',
427+
`${config.connectionDataSource}.default.${config.connectionTableName}`
428+
);
429+
queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
430+
queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation);
431+
queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName);
432+
queryStr = queryStr.replaceAll(/\s+/g, ' ');
433+
return queryStr;
434+
};
435+
348436
const addIntegration = async ({
349437
config,
350438
integration,
@@ -375,22 +463,20 @@ const addIntegration = async ({
375463
} else if (config.connectionType === 's3') {
376464
const http = coreRefs.http!;
377465

378-
const assets = await http.get(`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`);
466+
const assets: { data: ParsedIntegrationAsset[] } = await http.get(
467+
`${INTEGRATIONS_BASE}/repository/${integration.name}/assets`
468+
);
379469

380-
// Queries must exist because we disable s3 if they're not present
381470
for (const query of assets.data.filter(
382-
(a: ParsedIntegrationAsset): a is { type: 'query'; query: string; language: string } =>
471+
(a: ParsedIntegrationAsset): a is ParsedIntegrationAsset & { type: 'query' } =>
383472
a.type === 'query'
384473
)) {
385-
let queryStr = (query.query as string).replaceAll(
386-
'{table_name}',
387-
`${config.connectionDataSource}.default.${config.connectionTableName}`
388-
);
474+
// Skip any queries that have conditional workflows but aren't enabled
475+
if (query.workflows && !query.workflows.some((w) => config.enabledWorkflows.includes(w))) {
476+
continue;
477+
}
389478

390-
queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
391-
queryStr = queryStr.replaceAll('{s3_checkpoint_location}', config.checkpointLocation);
392-
queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName);
393-
queryStr = queryStr.replaceAll(/\s+/g, ' ');
479+
const queryStr = prepareQuery(query.query, config);
394480
const result = await runQuery(queryStr, config.connectionDataSource, sessionId);
395481
if (!result.ok) {
396482
setLoading(false);
@@ -400,15 +486,15 @@ const addIntegration = async ({
400486
sessionId = result.value.sessionId ?? sessionId;
401487
}
402488
// Once everything is ready, add the integration to the new datasource as usual
403-
// TODO determine actual values here after more about queries is known
404489
const res = await addIntegrationRequest(
405490
false,
406491
integration.name,
407492
config.displayName,
408493
integration,
409494
setCalloutLikeToast,
410495
config.displayName,
411-
`flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview`
496+
`flint_${config.connectionDataSource}_default_${config.connectionTableName}_mview`,
497+
config.enabledWorkflows
412498
);
413499
if (!res) {
414500
setLoading(false);
@@ -418,11 +504,14 @@ const addIntegration = async ({
418504
}
419505
};
420506

421-
const isConfigValid = (config: IntegrationSetupInputs): boolean => {
507+
const isConfigValid = (config: IntegrationSetupInputs, integration: IntegrationConfig): boolean => {
422508
if (config.displayName.length < 1 || config.connectionDataSource.length < 1) {
423509
return false;
424510
}
425511
if (config.connectionType === 's3') {
512+
if (integration.workflows && config.enabledWorkflows.length < 1) {
513+
return false;
514+
}
426515
return (
427516
config.connectionLocation.startsWith('s3://') && config.checkpointLocation.startsWith('s3://')
428517
);
@@ -477,7 +566,7 @@ export function SetupBottomBar({
477566
iconType="arrowRight"
478567
iconSide="right"
479568
isLoading={loading}
480-
disabled={!isConfigValid(config)}
569+
disabled={!isConfigValid(config, integration)}
481570
onClick={async () =>
482571
addIntegration({ integration, config, setLoading, setCalloutLikeToast })
483572
}
@@ -511,6 +600,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) {
511600
connectionLocation: '',
512601
checkpointLocation: '',
513602
connectionTableName: integration,
603+
enabledWorkflows: [],
514604
});
515605

516606
const [template, setTemplate] = useState({

‎server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json

+18-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,20 @@
88
"labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
99
"author": "OpenSearch",
1010
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info",
11+
"workflows": [
12+
{
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
17+
},
18+
{
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
23+
}
24+
],
1125
"statics": {
1226
"logo": {
1327
"annotation": "ELB Logo",
@@ -51,7 +65,8 @@
5165
"name": "aws_elb",
5266
"version": "1.0.0",
5367
"extension": "ndjson",
54-
"type": "savedObjectBundle"
68+
"type": "savedObjectBundle",
69+
"workflows": ["dashboards"]
5570
},
5671
{
5772
"name": "create_table",
@@ -63,7 +78,8 @@
6378
"name": "create_mv",
6479
"version": "1.0.0",
6580
"extension": "sql",
66-
"type": "query"
81+
"type": "query",
82+
"workflows": ["dashboards"]
6783
}
6884
],
6985
"sampleData": {

‎server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json

+18-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,20 @@
88
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
99
"author": "Haidong Wang",
1010
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
11+
"workflows": [
12+
{
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
17+
},
18+
{
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
23+
}
24+
],
1125
"statics": {
1226
"logo": {
1327
"annotation": "AWS VPC Logo",
@@ -47,7 +61,8 @@
4761
"name": "aws_vpc_flow",
4862
"version": "1.0.0",
4963
"extension": "ndjson",
50-
"type": "savedObjectBundle"
64+
"type": "savedObjectBundle",
65+
"workflows": ["dashboards"]
5166
},
5267
{
5368
"name": "create_table_vpc",
@@ -59,7 +74,8 @@
5974
"name": "create_mv_vpc",
6075
"version": "1.0.0",
6176
"extension": "sql",
62-
"type": "query"
77+
"type": "query",
78+
"workflows": ["dashboards"]
6379
}
6480
],
6581
"sampleData": {

‎server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

+18-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,20 @@
88
"labels": ["Observability", "Logs", "Flint S3"],
99
"author": "OpenSearch",
1010
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
11+
"workflows": [
12+
{
13+
"name": "queries",
14+
"label": "Queries (recommended)",
15+
"description": "Tables and pre-written queries for quickly getting insights on your data.",
16+
"enabled_by_default": true
17+
},
18+
{
19+
"name": "dashboards",
20+
"label": "Dashboards & Visualizations",
21+
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
22+
"enabled_by_default": false
23+
}
24+
],
1125
"statics": {
1226
"logo": {
1327
"annotation": "NginX Logo",
@@ -43,7 +57,8 @@
4357
"name": "nginx",
4458
"version": "1.0.0",
4559
"extension": "ndjson",
46-
"type": "savedObjectBundle"
60+
"type": "savedObjectBundle",
61+
"workflows": ["dashboards"]
4762
},
4863
{
4964
"name": "create_table",
@@ -55,7 +70,8 @@
5570
"name": "create_mv",
5671
"version": "1.0.0",
5772
"extension": "sql",
58-
"type": "query"
73+
"type": "query",
74+
"workflows": ["dashboards"]
5975
}
6076
],
6177
"sampleData": {

0 commit comments

Comments
 (0)