[Backport 2.x] Field mapping changes from opensearch-project#307 (opensearch-project#311)

amsiglan · AWSHurneyt · commit e70319fff77e · 2023-10-12T15:42:59.000-07:00
* Updated field mapping UX; disabled windows run for cypress (opensearch-project#307) * updated field mapping UX; disabled windows run for cypress Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated workflow file Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * added timestamp field to list of unmapped fields Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated cypress test Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> * updated cypress workflow Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com> Signed-off-by: AWSHurneyt <hurneyt@amazon.com>
diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml
@@ -13,7 +13,13 @@ env:
 jobs:
   tests:
     name: Run Cypress E2E tests
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        include:
+          - os: ubuntu-latest
+            cypress_cache_folder: ~/.cache/Cypress
+    runs-on: ${{ matrix.os }}
     env:
       # prevents extra Cypress installation progress messages
       CI: 1
@@ -25,55 +31,87 @@ jobs:
         with:
           # TODO: Parse this from security analytics plugin (https://github.com/opensearch-project/security-analytics/issues/170)
           java-version: 11
+
       - name: Checkout security analytics
         uses: actions/checkout@v2
         with:
           path: security-analytics
           repository: opensearch-project/security-analytics
           ref: ${{ env.SECURITY_ANALYTICS_BRANCH }}
+
       - name: Run opensearch with plugin
         run: |
           cd security-analytics
           ./gradlew run -Dopensearch.version=${{ env.OPENSEARCH_VERSION }} &
           sleep 300
-        # timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
-      - name: Checkout Security Analytics Dashboards plugin
-        uses: actions/checkout@v2
-        with:
-          path: security-analytics-dashboards-plugin
+        shell: bash
+
       - name: Checkout OpenSearch-Dashboards
         uses: actions/checkout@v2
         with:
           repository: opensearch-project/OpenSearch-Dashboards
           path: OpenSearch-Dashboards
           ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+
+      - name: Checkout Security Analytics Dashboards plugin
+        uses: actions/checkout@v2
+        with:
+          path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
+
       - name: Get node and yarn versions
         id: versions
         run: |
           echo "::set-output name=node_version::$(node -p "(require('./OpenSearch-Dashboards/package.json').engines.node).match(/[.0-9]+/)[0]")"
           echo "::set-output name=yarn_version::$(node -p "(require('./OpenSearch-Dashboards/package.json').engines.yarn).match(/[.0-9]+/)[0]")"
+
       - name: Setup node
         uses: actions/setup-node@v1
         with:
           node-version: ${{ steps.versions.outputs.node_version }}
           registry-url: 'https://registry.npmjs.org'
+
       - name: Install correct yarn version for OpenSearch-Dashboards
         run: |
           npm uninstall -g yarn
           echo "Installing yarn ${{ steps.versions_step.outputs.yarn_version }}"
           npm i -g yarn@${{ steps.versions.outputs.yarn_version }}
+
       - name: Bootstrap plugin/OpenSearch-Dashboards
         run: |
-          mkdir -p OpenSearch-Dashboards/plugins
-          mv security-analytics-dashboards-plugin OpenSearch-Dashboards/plugins
           cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
           yarn osd bootstrap
+
       - name: Run OpenSearch-Dashboards server
         run: |
           cd OpenSearch-Dashboards
           yarn start --no-base-path --no-watch &
-          sleep 300
-        # timeout 300 bash -c 'while [[ "$(curl -s localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done'
+        shell: bash
+
+      - name: Sleep until OSD server starts
+        run: sleep 300
+        shell: bash
+      
+      - name: Install Cypress
+        run: |
+          cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
+          # This will install Cypress in case the binary is missing which can happen on Windows and Mac
+          # If the binary exists, this will exit quickly so it should not be an expensive operation
+          npx cypress install
+        shell: bash
+
+      - name: Get Cypress version
+        id: cypress_version
+        run: |
+          cd OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin
+          echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
+
+      - name: Cache Cypress
+        id: cache-cypress
+        uses: actions/cache@v2
+        with:
+          path: ${{ matrix.cypress_cache_folder }}
+          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
+        
       # for now just chrome, use matrix to do all browsers later
       - name: Cypress tests
         uses: cypress-io/github-action@v2
@@ -82,15 +120,19 @@ jobs:
           command: yarn run cypress run
           wait-on: 'http://localhost:5601'
           browser: chrome
+        env:
+          CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}      
+
       # Screenshots are only captured on failure, will change this once we do visual regression tests
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
-          name: cypress-screenshots
+          name: cypress-screenshots-${{ matrix.os }}
           path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin/cypress/screenshots
+
       # Test run video was always captured, so this action uses "always()" condition
       - uses: actions/upload-artifact@v1
         if: always()
         with:
-          name: cypress-videos
+          name: cypress-videos-${{ matrix.os }}
           path: OpenSearch-Dashboards/plugins/security-analytics-dashboards-plugin/cypress/videos
diff --git a/cypress/integration/1_detectors.spec.js b/cypress/integration/1_detectors.spec.js
@@ -69,7 +69,7 @@ describe('Detectors', () => {
     cy.get('button').contains('Next').click({ force: true });
 
     // Check that correct page now showing
-    cy.contains('Required field mappings');
+    cy.contains('Configure field mapping');
 
     // Select appropriate names to map fields to
     for (let field_name in sample_field_mappings.properties) {
diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx b/public/pages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx
@@ -5,12 +5,20 @@
 
 import React, { Component } from 'react';
 import { RouteComponentProps } from 'react-router-dom';
-import { EuiSpacer, EuiTitle, EuiText } from '@elastic/eui';
+import {
+  EuiSpacer,
+  EuiTitle,
+  EuiText,
+  EuiCallOut,
+  EuiAccordion,
+  EuiHorizontalRule,
+  EuiPanel,
+} from '@elastic/eui';
 import FieldMappingsTable from '../components/RequiredFieldMapping';
 import { createDetectorSteps } from '../../../utils/constants';
 import { ContentPanel } from '../../../../../components/ContentPanel';
 import { Detector, FieldMapping } from '../../../../../../models/interfaces';
-import { EMPTY_FIELD_MAPPINGS } from '../utils/constants';
+import { EMPTY_FIELD_MAPPINGS_VIEW } from '../utils/constants';
 import { DetectorCreationStep } from '../../../models/types';
 import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces';
 import FieldMappingService from '../../../../../services/FieldMappingService';
@@ -49,7 +57,7 @@ export default class ConfigureFieldMapping extends Component<
     });
     this.state = {
       loading: props.loading || false,
-      mappingsData: EMPTY_FIELD_MAPPINGS,
+      mappingsData: EMPTY_FIELD_MAPPINGS_VIEW,
       createdMappings,
       invalidMappingFieldNames: [],
     };
@@ -70,17 +78,21 @@ export default class ConfigureFieldMapping extends Component<
       Object.keys(mappingsView.response.properties).forEach((ruleFieldName) => {
         existingMappings[ruleFieldName] = mappingsView.response.properties[ruleFieldName].path;
       });
-      this.setState({ createdMappings: existingMappings, mappingsData: mappingsView.response });
+      this.setState({
+        createdMappings: existingMappings,
+        mappingsData: {
+          ...mappingsView.response,
+          unmapped_field_aliases: [
+            'timestamp',
+            ...(mappingsView.response.unmapped_field_aliases || []),
+          ],
+        },
+      });
       this.updateMappingSharedState(existingMappings);
     }
     this.setState({ loading: false });
   };
 
-  validateMappings(mappings: ruleFieldToIndexFieldMap): boolean {
-    // TODO: Implement validation
-    return true; //allFieldsMapped; // && allAliasesUnique;
-  }
-
   /**
    * Returns the fieldName(s) that have duplicate alias assigned to them
    */
@@ -110,8 +122,7 @@ export default class ConfigureFieldMapping extends Component<
       invalidMappingFieldNames: invalidMappingFieldNames,
     });
     this.updateMappingSharedState(newMappings);
-    const mappingsValid = this.validateMappings(newMappings);
-    this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, mappingsValid);
+    this.props.updateDataValidState(DetectorCreationStep.CONFIGURE_FIELD_MAPPING, true);
   };
 
   updateMappingSharedState = (createdMappings: ruleFieldToIndexFieldMap) => {
@@ -126,42 +137,52 @@ export default class ConfigureFieldMapping extends Component<
   };
 
   render() {
-    const { isEdit } = this.props;
     const { loading, mappingsData, createdMappings, invalidMappingFieldNames } = this.state;
     const existingMappings: ruleFieldToIndexFieldMap = {
       ...createdMappings,
     };
-    const ruleFields = [...(mappingsData.unmapped_field_aliases || [])];
-    const indexFields = [...(mappingsData.unmapped_index_fields || [])];
 
+    // read only data
+    const mappedRuleFields: string[] = [];
+    const mappedLogFields: string[] = [];
     Object.keys(mappingsData.properties).forEach((ruleFieldName) => {
-      existingMappings[ruleFieldName] = mappingsData.properties[ruleFieldName].path;
-      ruleFields.unshift(ruleFieldName);
-      indexFields.unshift(mappingsData.properties[ruleFieldName].path);
+      mappedRuleFields.unshift(ruleFieldName);
+      mappedLogFields.unshift(mappingsData.properties[ruleFieldName].path);
     });
 
+    // edit data
+    const ruleFields = [...(mappingsData.unmapped_field_aliases || [])];
+    const indexFields = [...(mappingsData.unmapped_index_fields || [])];
+
     return (
       <div>
-        {!isEdit && (
-          <>
-            <EuiTitle size={'m'}>
-              <h3>{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}</h3>
-            </EuiTitle>
+        <EuiTitle size={'m'}>
+          <h3>{createDetectorSteps[DetectorCreationStep.CONFIGURE_FIELD_MAPPING].title}</h3>
+        </EuiTitle>
 
-            <EuiText size="s" color="subdued">
-              To perform threat detection, known field names from your log data source are
-              automatically mapped to rule field names. Additional fields that may require manual
-              mapping will be shown below.
-            </EuiText>
+        <EuiText size="s" color="subdued">
+          To perform threat detection, known field names from your log data source are automatically
+          mapped to rule field names. Additional fields that may require manual mapping will be
+          shown below.
+        </EuiText>
 
-            <EuiSpacer size={'m'} />
-          </>
-        )}
+        <EuiSpacer size={'m'} />
 
-        {ruleFields.length > 0 && (
+        {ruleFields.length > 0 ? (
           <>
-            <ContentPanel title={`Required field mappings (${ruleFields.length})`} titleSize={'m'}>
+            <EuiCallOut
+              title={`${ruleFields.length} rule fields may need manual mapping`}
+              color={'warning'}
+            >
+              <p>
+                To generate accurate findings, we recommend mapping the following security rules
+                fields with the log field from your data source.
+              </p>
+            </EuiCallOut>
+            <EuiSpacer size={'m'} />
+            <ContentPanel title={`Manual field mappings (${ruleFields.length})`} titleSize={'m'}>
               <FieldMappingsTable<MappingViewType.Edit>
+                {...this.props}
                 loading={loading}
                 ruleFields={ruleFields}
                 indexFields={indexFields}
@@ -171,12 +192,48 @@ export default class ConfigureFieldMapping extends Component<
                   invalidMappingFieldNames,
                   onMappingCreation: this.onMappingCreation,
                 }}
-                {...this.props}
               />
             </ContentPanel>
             <EuiSpacer size={'m'} />
           </>
+        ) : (
+          <>
+            <EuiCallOut title={'We have automatically mapped all fields'} color={'success'}>
+              <p>
+                Your data source(s) have been mapped with all security rule fields. No action is
+                needed.
+              </p>
+            </EuiCallOut>
+            <EuiSpacer size={'m'} />
+          </>
         )}
+
+        <EuiPanel>
+          <EuiAccordion
+            buttonContent={
+              <div data-test-subj="mapped-fields-btn">
+                <EuiTitle>
+                  <h4>{`View mapped fields (${mappedRuleFields.length})`}</h4>
+                </EuiTitle>
+              </div>
+            }
+            buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
+            id={'mappedFieldsAccordion'}
+            initialIsOpen={false}
+          >
+            <EuiHorizontalRule margin={'xs'} />
+            <FieldMappingsTable<MappingViewType.Readonly>
+              {...this.props}
+              loading={loading}
+              ruleFields={mappedRuleFields}
+              indexFields={mappedLogFields}
+              mappingProps={{
+                type: MappingViewType.Readonly,
+              }}
+            />
+          </EuiAccordion>
+        </EuiPanel>
+        <EuiSpacer size={'m'} />
       </div>
     );
   }
diff --git a/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts b/public/pages/CreateDetector/components/ConfigureFieldMapping/utils/constants.ts
@@ -3,15 +3,22 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { GetFieldMappingViewResponse } from '../../../../../../server/models/interfaces';
+import {
+  FieldMappingPropertyMap,
+  GetFieldMappingViewResponse,
+} from '../../../../../../server/models/interfaces';
 
 export const STATUS_ICON_PROPS = {
   unmapped: { type: 'alert', color: 'danger' },
   mapped: { type: 'checkInCircleFilled', color: 'success' },
 };
 
-export const EMPTY_FIELD_MAPPINGS: GetFieldMappingViewResponse = {
+export const EMPTY_FIELD_MAPPINGS_VIEW: GetFieldMappingViewResponse = {
   properties: {},
   unmapped_field_aliases: [],
   unmapped_index_fields: [],
 };
+
+export const EMPTY_FIELD_MAPPINGS: FieldMappingPropertyMap = {
+  properties: {},
+};
diff --git a/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx b/public/pages/Detectors/components/FieldMappingsView/FieldMappingsView.tsx
@@ -52,10 +52,10 @@ export const FieldMappingsView: React.FC<FieldMappingsViewProps> = ({
     async (indexName: string) => {
       const getMappingRes = await services?.fieldMappingService.getMappings(indexName);
       if (getMappingRes?.ok) {
-        const mappings = getMappingRes.response[detector.detector_type.toLowerCase()];
-        if (mappings) {
+        const mappingsData = getMappingRes.response[indexName];
+        if (mappingsData) {
           let items: FieldMappingsTableItem[] = [];
-          Object.entries(mappings.mappings.properties).forEach((entry) => {
+          Object.entries(mappingsData.mappings.properties).forEach((entry) => {
             items.push({
               ruleFieldName: entry[0],
               logFieldName: entry[1].path,
diff --git a/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx b/public/pages/Detectors/components/UpdateFieldMappings/UpdateFieldMappings.tsx
diff --git a/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx b/public/pages/Detectors/containers/FieldMappings/EditFieldMapping.tsx
