Updated field mapping UX; disabled windows run for cypress (opensearch-project#307)

amsiglan · amsiglan · commit 3bca7788df1b · 2023-01-10T22:59:22.000+05:30
* updated field mapping UX; disabled windows run for cypress

Signed-off-by: Amardeepsingh Siglani &lt;amardeep7194@gmail.com&gt;

* updated workflow file

Signed-off-by: Amardeepsingh Siglani &lt;amardeep7194@gmail.com&gt;

* added timestamp field to list of unmapped fields

Signed-off-by: Amardeepsingh Siglani &lt;amardeep7194@gmail.com&gt;

* updated cypress test

Signed-off-by: Amardeepsingh Siglani &lt;amardeep7194@gmail.com&gt;

Signed-off-by: Amardeepsingh Siglani &lt;amardeep7194@gmail.com&gt;
diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml
@@ -13,7 +13,13 @@ env:
 jobs:
   tests:
     name: Run Cypress E2E tests
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        include:
+          - os: ubuntu-latest
+            cypress_cache_folder: ~/.cache/Cypress
+    runs-on: ${{ matrix.os }}
     env:
       # prevents extra Cypress installation progress messages
       CI: 1
@@ -72,8 +78,33 @@ jobs:
         run: |
           cd OpenSearch-Dashboards
           yarn start --no-base-path --no-watch &
-          sleep 300
-        # timeout 300 bash -c 'while [[ "$(curl -s localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done'
+        shell: bash
+
+      - name: Sleep until OSD server starts
+        run: sleep 300
+        shell: bash
+      
+      - name: Install Cypress
+        run: |
+          cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
+          # This will install Cypress in case the binary is missing which can happen on Windows and Mac
+          # If the binary exists, this will exit quickly so it should not be an expensive operation
+          npx cypress install
+        shell: bash
+
+      - name: Get Cypress version
+        id: cypress_version
+        run: |
+          cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
+          echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
+
+      - name: Cache Cypress
+        id: cache-cypress
+        uses: actions/cache@v2
+        with:
+          path: ${{ matrix.cypress_cache_folder }}
+          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
+        
       # for now just chrome, use matrix to do all browsers later
       - name: Cypress tests
         uses: cypress-io/github-action@v2
diff --git a/cypress/integration/1_detectors.spec.js b/cypress/integration/1_detectors.spec.js
@@ -69,7 +69,7 @@ describe('Detectors', () => {
     cy.get('button').contains('Next').click({ force: true });
 
     // Check that correct page now showing
-    cy.contains('Required field mappings');
+    cy.contains('Configure field mapping');
 
     // Select appropriate names to map fields to
     for (let field_name in sample_field_mappings.properties) {
diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx b/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx
@@ -5,12 +5,20 @@
 
 import React, { Component } from 'react';
 import { RouteComponentProps } from 'react-router-dom';
-import { EuiSpacer, EuiTitle, EuiText } from '@elastic/eui';
+import {
+  EuiSpacer,
+  EuiTitle,
+  EuiText,
+  EuiCallOut,
+  EuiAccordion,
+  EuiHorizontalRule,
+  EuiPanel,
+} from '@elastic/eui';
 import FieldMappingsTable from '../components/RequiredFieldMapping';
 import { createDetectorSteps } from '../../../utils/constants';
 import { ContentPanel } from '../../../../../components/ContentPanel';
 import { Detector, FieldMapping } from '../../../../../../models/interfaces';
-import { EMPTY_FIELD_MAPPINGS } from '../utils/constants';
+import { EMPTY_FIELD_MAPPINGS_VIEW } from '../utils/constants';
 import { DetectorCreationStep } from '../../../models/types';
 import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces';
 import FieldMappingService from '../../../../../services/FieldMappingService';
@@ -49,7 +57,7 @@ export default class ConfigureFieldMapping extends Component<
     });
     this.state = {
       loading: props.loading || false,
-      mappingsData: EMPTY_FIELD_MAPPINGS,
+      mappingsData: EMPTY_FIELD_MAPPINGS_VIEW,
       createdMappings,
       invalidMappingFieldNames: [],
     };
@@ -70,17 +78,21 @@ export default class ConfigureFieldMapping extends Component<
       Object.keys(mappingsView.response.properties).forEach((ruleFieldName) => {
         existingMappings[ruleFieldName] = mappingsView.response.properties[ruleFieldName].path;
       });
-      this.setState({ createdMappings: existingMappings, mappingsData: mappingsView.response });
+      this.setState({
+        createdMappings: existingMappings,
+        mappingsData: {
+          ...mappingsView.response,
+          unmapped_field_aliases: [
+            'timestamp',
+            ...(mappingsView.response.unmapped_field_aliases || []),
+          ],
+        },
+      });
       this.updateMappingSharedState(existingMappings);
     }
     this.setState({ loading: false });
   };
 
-  validateMappings(mappings: ruleFieldToIndexFieldMap): boolean {
-    // TODO: Implement validation
-    return true; //allFieldsMapped; // && allAliasesUnique;
-  }
-
   /**
    * Returns the fieldName(s) that have duplicate alias assigned to them
    */
@@ -110,8 +122,7 @@ export default class ConfigureFieldMapping extends Component<
       invalidMappingFieldNames: invalidMappingFieldNames,
     });
     this.updateMappingSharedState(newMappings);
-    const mappingsValid = this.validateMappings(newMappings);
-    this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, mappingsValid);
+    this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, true);
   };
 
   updateMappingSharedState = (createdMappings: ruleFieldToIndexFieldMap) => {
@@ -126,42 +137,52 @@ export default class ConfigureFieldMapping extends Component<
   };
 
   render() {
-    const { isEdit } = this.props;
     const { loading, mappingsData, createdMappings, invalidMappingFieldNames } = this.state;
     const existingMappings: ruleFieldToIndexFieldMap = {
       ...createdMappings,
     };
-    const ruleFields = [...(mappingsData.unmapped_field_aliases || [])];
-    const indexFields = [...(mappingsData.unmapped_index_fields || [])];
 
+    // read only data
+    const mappedRuleFields: string[] = [];
+    const mappedLogFields: string[] = [];
     Object.keys(mappingsData.properties).forEach((ruleFieldName) => {
-      existingMappings[ruleFieldName] = mappingsData.properties[ruleFieldName].path;
-      ruleFields.unshift(ruleFieldName);
-      indexFields.unshift(mappingsData.properties[ruleFieldName].path);
+      mappedRuleFields.unshift(ruleFieldName);
+      mappedLogFields.unshift(mappingsData.properties[ruleFieldName].path);
     });
 
+    // edit data
+    const ruleFields = [...(mappingsData.unmapped_field_aliases || [])];
+    const indexFields = [...(mappingsData.unmapped_index_fields || [])];
+
     return (
       <div>
-        {!isEdit && (
-          <>
-            <EuiTitle size={'m'}>
-              <h3>{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}</h3>
-            </EuiTitle>
+        <EuiTitle size={'m'}>
+          <h3>{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}</h3>
+        </EuiTitle>
 
-            <EuiText size="s" color="subdued">
-              To perform threat detection, known field names from your log data source are
-              automatically mapped to rule field names. Additional fields that may require manual
-              mapping will be shown below.
-            </EuiText>
+        <EuiText size="s" color="subdued">
+          To perform threat detection, known field names from your log data source are automatically
+          mapped to rule field names. Additional fields that may require manual mapping will be
+          shown below.
+        </EuiText>
 
-            <EuiSpacer size={'m'} />
-          </>
-        )}
+        <EuiSpacer size={'m'} />
 
-        {ruleFields.length > 0 && (
+        {ruleFields.length > 0 ? (
           <>
-            <ContentPanel title={`Required field mappings (${ruleFields.length})`} titleSize={'m'}>
+            <EuiCallOut
+              title={`${ruleFields.length} rule fields may need manual mapping`}
+              color={'warning'}
+            >
+              <p>
+                To generate accurate findings, we recommend mapping the following security rules
+                fields with the log field from your data source.
+              </p>
+            </EuiCallOut>
+            <EuiSpacer size={'m'} />
+            <ContentPanel title={`Manual field mappings (${ruleFields.length})`} titleSize={'m'}>
               <FieldMappingsTable<MappingViewType.Edit>
+                {...this.props}
                 loading={loading}
                 ruleFields={ruleFields}
                 indexFields={indexFields}
@@ -171,12 +192,48 @@ export default class ConfigureFieldMapping extends Component<
                   invalidMappingFieldNames,
                   onMappingCreation: this.onMappingCreation,
                 }}
-                {...this.props}
               />
             </ContentPanel>
             <EuiSpacer size={'m'} />
           </>
+        ) : (
+          <>
+            <EuiCallOut title={'We have automatically mapped all fields'} color={'success'}>
+              <p>
+                Your data source(s) have been mapped with all security rule fields. No action is
+                needed.
+              </p>
+            </EuiCallOut>
+            <EuiSpacer size={'m'} />
+          </>
         )}
+
+        <EuiPanel>
+          <EuiAccordion
+            buttonContent={
+              <div data-test-subj="mapped-fields-btn">
+                <EuiTitle>
+                  <h4>{`View mapped fields (${mappedRuleFields.length})`}</h4>
+                </EuiTitle>
+              </div>
+            }
+            buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
+            id={'mappedFieldsAccordion'}
+            initialIsOpen={false}
+          >
+            <EuiHorizontalRule margin={'xs'} />
+            <FieldMappingsTable<MappingViewType.Readonly>
+              {...this.props}
+              loading={loading}
+              ruleFields={mappedRuleFields}
+              indexFields={mappedLogFields}
+              mappingProps={{
+                type: MappingViewType.Readonly,
+              }}
+            />
+          </EuiAccordion>
+        </EuiPanel>
+        <EuiSpacer size={'m'} />
       </div>
     );
   }
diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts b/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts
@@ -3,15 +3,22 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces';
+import {
+  FieldMappingPropertyMap,
+  GetFieldMappingViewResponse,
+} from '../../../../../../server/models/interfaces';
 
 export const STATUS_ICON_PROPS = {
   unmapped: { type: 'alert', color: 'danger' },
   mapped: { type: 'checkInCircleFilled', color: 'success' },
 };
 
-export const EMPTY_FIELD_MAPPINGS: GetFieldMappingViewResponse = {
+export const EMPTY_FIELD_MAPPINGS_VIEW: GetFieldMappingViewResponse = {
   properties: {},
   unmapped_field_aliases: [],
   unmapped_index_fields: [],
 };
+
+export const EMPTY_FIELD_MAPPINGS: FieldMappingPropertyMap = {
+  properties: {},
+};
diff --git a/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx b/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx
@@ -52,10 +52,10 @@ export const FieldMappingsView: React.FC<FieldMappingsViewProps> = ({
     async (indexName: string) => {
       const getMappingRes = await services?.fieldMappingService.getMappings(indexName);
       if (getMappingRes?.ok) {
-        const mappings = getMappingRes.response[detector.detector_type.toLowerCase()];
-        if (mappings) {
+        const mappingsData = getMappingRes.response[indexName];
+        if (mappingsData) {
           let items: FieldMappingsTableItem[] = [];
-          Object.entries(mappings.mappings.properties).forEach((entry) => {
+          Object.entries(mappingsData.mappings.properties).forEach((entry) => {
             items.push({
               ruleFieldName: entry[0],
               logFieldName: entry[1].path,
diff --git a/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx b/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx
@@ -6,7 +6,6 @@
 import React, { Component } from 'react';
 import { RouteComponentProps } from 'react-router-dom';
 import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiTitle } from '@elastic/eui';
-import ConfigureFieldMapping from '../../../CreateDetector/components/ConfigureFieldMapping';
 import { Detector, FieldMapping } from '../../../../../models/interfaces';
 import FieldMappingService from '../../../../services/FieldMappingService';
 import { DetectorHit, SearchDetectorsResponse } from '../../../../../server/models/interfaces';
@@ -15,6 +14,7 @@ import { DetectorsService } from '../../../../services';
 import { ServerResponse } from '../../../../../server/models/types';
 import { NotificationsStart } from 'opensearch-dashboards/public';
 import { errorNotificationToast, successNotificationToast } from '../../../../utils/helpers';
+import EditFieldMappings from '../../containers/FieldMappings/EditFieldMapping';
 
 export interface UpdateFieldMappingsProps
   extends RouteComponentProps<any, any, { detectorHit: DetectorHit }> {
@@ -159,14 +159,12 @@ export default class UpdateFieldMappings extends Component<
         <EuiSpacer size={'xxl'} />
 
         {!loading && (
-          <ConfigureFieldMapping
+          <EditFieldMappings
             {...this.props}
-            isEdit={true}
             detector={detector}
             fieldMappings={fieldMappings}
             filedMappingService={filedMappingService}
             replaceFieldMappings={this.replaceFieldMappings}
-            updateDataValidState={() => {}}
             loading={loading}
           />
         )}
diff --git a/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx b/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx
