Add support for running CI with security (#263)

ohltyler · web-flow · commit 455e3f8b65f2 · 2020-07-29T19:40:41.000-07:00
diff --git a/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts b/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts
@@ -68,9 +68,7 @@ context('AD Dashboard', () => {
     cy.get('[data-test-subj=comboBoxToggleListButton]')
       .first()
       .click({ force: true });
-    cy.get('.euiFilterSelectItem')
-      .first()
-      .click({ force: true });
+    cy.get('.euiFilterSelectItem').first().click({ force: true });
     cy.get('.euiPageSideBar').click({ force: true });
 
     cy.contains('feature-required-detector'); // first one in the list returned by multiple_detectors_response.json
@@ -89,9 +87,7 @@ context('AD Dashboard', () => {
     cy.get('[data-test-subj=comboBoxToggleListButton]')
       .eq(1)
       .click({ force: true });
-    cy.get('.euiFilterSelectItem')
-      .first()
-      .click({ force: true });
+    cy.get('.euiFilterSelectItem').first().click({ force: true });
     cy.get('.euiPageSideBar').click({ force: true });
 
     cy.contains('stopped-detector'); // because stopped is the first item in the detector state dropdown
diff --git a/.cypress/integration/ad/detectorList/detector_list.spec.ts b/.cypress/integration/ad/detectorList/detector_list.spec.ts
@@ -89,9 +89,7 @@ context('Detector list', () => {
     cy.mockGetDetectorOnAction('single_stopped_detector_response.json', () => {
       cy.visit(buildAdAppUrl(DETECTORS));
     });
-    cy.get('.euiCheckbox__input')
-      .last()
-      .click({ force: true });
+    cy.get('.euiCheckbox__input').last().click({ force: true });
     cy.get('[data-test-subj=listActionsButton]').click({ force: true });
     cy.get('[data-test-subj=startDetectors]').click({ force: true });
     cy.contains('The following detectors will begin initializing.');
@@ -110,9 +108,7 @@ context('Detector list', () => {
     cy.mockGetDetectorOnAction('single_running_detector_response.json', () => {
       cy.visit(buildAdAppUrl(DETECTORS));
     });
-    cy.get('.euiCheckbox__input')
-      .last()
-      .click({ force: true });
+    cy.get('.euiCheckbox__input').last().click({ force: true });
     cy.get('[data-test-subj=listActionsButton]').click({ force: true });
     cy.get('[data-test-subj=stopDetectors]').click({ force: true });
     cy.contains('The following detectors will be stopped.');
@@ -131,9 +127,7 @@ context('Detector list', () => {
     cy.mockGetDetectorOnAction('single_stopped_detector_response.json', () => {
       cy.visit(buildAdAppUrl(DETECTORS));
     });
-    cy.get('.euiCheckbox__input')
-      .last()
-      .click({ force: true });
+    cy.get('.euiCheckbox__input').last().click({ force: true });
     cy.get('[data-test-subj=listActionsButton]').click({ force: true });
     cy.get('[data-test-subj=deleteDetectors]').click({ force: true });
     cy.contains(
@@ -183,9 +177,7 @@ context('Detector list', () => {
     cy.get('[data-test-subj=comboBoxToggleListButton]')
       .first()
       .click({ force: true });
-    cy.get('.euiFilterSelectItem')
-      .first()
-      .click({ force: true });
+    cy.get('.euiFilterSelectItem').first().click({ force: true });
     cy.get('.euiPageSideBar').click({ force: true });
 
     cy.contains('stopped-detector'); // because stopped is the first item in the detector state dropdown
diff --git a/.cypress/support/commands.ts b/.cypress/support/commands.ts
@@ -23,7 +23,29 @@ import {
 } from '../utils/constants';
 import { buildAdApiUrl } from '../utils/helpers';
 
-Cypress.Commands.add('mockGetDetectorOnAction', function(
+Cypress.Commands.overwrite('visit', (orig, url, options) => {
+  if (Cypress.env('SECURITY_ENABLED')) {
+    let newOptions = options;
+    if (options) {
+      newOptions['auth'] = {
+        username: 'admin',
+        password: 'admin',
+      };
+    } else {
+      newOptions = {
+        auth: {
+          username: 'admin',
+          password: 'admin',
+        },
+      };
+    }
+    orig(url, newOptions);
+  } else {
+    orig(url, options);
+  }
+});
+
+Cypress.Commands.add('mockGetDetectorOnAction', function (
   fixtureFileName: string,
   funcMockedOn: VoidFunction
 ) {
@@ -39,7 +61,7 @@ Cypress.Commands.add('mockGetDetectorOnAction', function(
   cy.wait('@getDetectors');
 });
 
-Cypress.Commands.add('mockCreateDetectorOnAction', function(
+Cypress.Commands.add('mockCreateDetectorOnAction', function (
   fixtureFileName: string,
   funcMockedOn: VoidFunction
 ) {
@@ -55,7 +77,7 @@ Cypress.Commands.add('mockCreateDetectorOnAction', function(
   cy.wait('@createDetector');
 });
 
-Cypress.Commands.add('mockSearchIndexOnAction', function(
+Cypress.Commands.add('mockSearchIndexOnAction', function (
   fixtureFileName: string,
   funcMockedOn: VoidFunction
 ) {
@@ -71,7 +93,7 @@ Cypress.Commands.add('mockSearchIndexOnAction', function(
   cy.wait('@getIndices');
 });
 
-Cypress.Commands.add('mockSearchOnAction', function(
+Cypress.Commands.add('mockSearchOnAction', function (
   fixtureFileName: string,
   funcMockedOn: VoidFunction
 ) {
@@ -85,7 +107,7 @@ Cypress.Commands.add('mockSearchOnAction', function(
   cy.wait('@searchES');
 });
 
-Cypress.Commands.add('mockGetIndexMappingsOnAction', function(
+Cypress.Commands.add('mockGetIndexMappingsOnAction', function (
   fixtureFileName: string,
   funcMockedOn: VoidFunction
 ) {
@@ -101,7 +123,7 @@ Cypress.Commands.add('mockGetIndexMappingsOnAction', function(
   cy.wait('@getMappings');
 });
 
-Cypress.Commands.add('mockStartDetectorOnAction', function(
+Cypress.Commands.add('mockStartDetectorOnAction', function (
   fixtureFileName: string,
   detectorId: string,
   funcMockedOn: VoidFunction
@@ -118,7 +140,7 @@ Cypress.Commands.add('mockStartDetectorOnAction', function(
   cy.wait('@startDetector');
 });
 
-Cypress.Commands.add('mockStopDetectorOnAction', function(
+Cypress.Commands.add('mockStopDetectorOnAction', function (
   fixtureFileName: string,
   detectorId: string,
   funcMockedOn: VoidFunction
@@ -135,7 +157,7 @@ Cypress.Commands.add('mockStopDetectorOnAction', function(
   cy.wait('@stopDetector');
 });
 
-Cypress.Commands.add('mockDeleteDetectorOnAction', function(
+Cypress.Commands.add('mockDeleteDetectorOnAction', function (
   fixtureFileName: string,
   detectorId: string,
   funcMockedOn: VoidFunction
diff --git a/.github/configurations/docker-compose.yml b/.github/configurations/docker-compose.yml
@@ -0,0 +1,46 @@
+version: '3'
+services:
+  odfe-node1:
+    image: odfe-ad:test
+    container_name: odfe-node1
+    environment:
+      - cluster.name=odfe-cluster
+      - node.name=odfe-node1
+      - discovery.seed_hosts=odfe-node1
+      - cluster.initial_master_nodes=odfe-node1
+      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
+      - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - odfe-data1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+      - 9600:9600 # required for Performance Analyzer
+    expose:
+      - '9200'
+    networks:
+      - odfe-net
+  kibana:
+    image: odfe-ad-kibana:test
+    container_name: odfe-kibana
+    ports:
+      - 5601:5601
+    expose:
+      - '5601'
+    environment:
+      ELASTICSEARCH_URL: https://odfe-node1:9200
+      ELASTICSEARCH_HOSTS: https://odfe-node1:9200
+    networks:
+      - odfe-net
+
+volumes:
+  odfe-data1:
+
+networks:
+  odfe-net:
diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml
@@ -3,7 +3,8 @@ on:
   push:
     tags:
       - v*
-
+env:
+  KIBANA_VERSION: 7.8.0
 jobs:
   Build-and-upload:
     name: Build and upload artifacts
@@ -20,7 +21,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: opendistro-for-elasticsearch/kibana-oss
-          ref: 7.8.0
+          ref: ${{ env.KIBANA_VERSION }}
           token: ${{ secrets.KIBANA_OSS_ACCESS }}
           path: kibana
 
diff --git a/.github/workflows/e2e-tests-workflow.yml b/.github/workflows/e2e-tests-workflow.yml
@@ -2,10 +2,99 @@ name: E2E tests workflow
 on:
   push:
     branches:
-      - master
+      - securityCI
+env:
+  KIBANA_VERSION: 7.8.0
+  ODFE_VERSION: 1.9.0
 jobs:
-  tests:
-    name: Run e2e tests
+  test-with-security:
+    name: Run e2e tests with security
+    strategy:
+      matrix:
+        os: [ubuntu-16.04] # use ubuntu-16.04 as required by cypress: https://github.com/marketplace/actions/cypress-io#important
+        java: [14]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Kibana
+        uses: actions/checkout@v2
+        with:
+          repository: opendistro-for-elasticsearch/kibana-oss
+          ref: ${{ env.KIBANA_VERSION }}
+          token: ${{ secrets.KIBANA_OSS_ACCESS }}
+          path: kibana
+
+      - name: Get node and yarn versions
+        id: versions_step
+        run: |
+          echo "::set-output name=node_version::$(node -p "(require('./kibana/package.json').engines.node).match(/[.0-9]+/)[0]")"
+          echo "::set-output name=yarn_version::$(node -p "(require('./kibana/package.json').engines.yarn).match(/[.0-9]+/)[0]")"
+
+      - name: Setup node
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ steps.versions_step.outputs.node_version }}
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install correct yarn version for Kibana
+        run: |
+          npm uninstall -g yarn
+          echo "Installing yarn ${{ steps.versions_step.outputs.yarn_version }}"
+          npm i -g yarn@${{ steps.versions_step.outputs.yarn_version }}
+
+      - name: Checkout Anomaly Detection Kibana plugin
+        uses: actions/checkout@v2
+        with:
+          path: kibana/plugins/anomaly-detection-kibana-plugin
+
+      - name: Bootstrap the plugin
+        run: |
+          cd kibana/plugins/anomaly-detection-kibana-plugin
+          yarn kbn bootstrap
+
+      - name: Build the artifact
+        run: |
+          cd kibana/plugins/anomaly-detection-kibana-plugin
+          yarn build
+
+      - name: Create tagged images for ES and Kibana
+        run: |
+          cd kibana/plugins/anomaly-detection-kibana-plugin
+          odfe_version=$ODFE_VERSION
+          plugin_version=$(node -pe "require('./package.json').version")
+          echo odfe version: $odfe_version
+          echo plugin version: $plugin_version
+          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$odfe_version
+          then
+            ## Populate the Dockerfiles
+            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$odfe_version" >> Dockerfile-AD
+            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$odfe_version" >> Dockerfile-AD-Kibana
+            echo "COPY build/opendistro-anomaly-detection-kibana-$plugin_version.zip ." >> Dockerfile-AD-Kibana
+            ## Uninstall existing AD artifact and install new one
+            echo "RUN if [ -d /usr/share/kibana/plugins/opendistro-anomaly-detection-kibana ]; then /usr/share/kibana/bin/kibana-plugin remove opendistro-anomaly-detection-kibana; fi" >> Dockerfile-AD-Kibana
+            echo "RUN bin/kibana-plugin install file:///usr/share/kibana/opendistro-anomaly-detection-kibana-$plugin_version.zip;" >> Dockerfile-AD-Kibana
+
+            ## Create the tagged images
+            docker build -f ./Dockerfile-AD -t odfe-ad:test .
+            docker build -f ./Dockerfile-AD-Kibana -t odfe-ad-kibana:test .
+          fi
+          docker images
+
+      - name: Start ES and Kibana
+        run: |
+          cd kibana/plugins/anomaly-detection-kibana-plugin/.github/configurations
+          ## Need to increase max map count for running the docker container
+          sudo sysctl -w vm.max_map_count=262144
+          docker-compose up -d
+          sleep 180
+
+      - name: Run e2e tests
+        uses: cypress-io/github-action@v1
+        with:
+          working-directory: kibana/plugins/anomaly-detection-kibana-plugin
+          command: yarn cy:run --env SECURITY_ENABLED=true
+
+  test-without-security:
+    name: Run e2e tests without security
     strategy:
       matrix:
         os: [ubuntu-16.04] # use ubuntu-16.04 as required by cypress: https://github.com/marketplace/actions/cypress-io#important
@@ -14,11 +103,11 @@ jobs:
     steps:
       - name: Pull and Run Docker
         run: |
-          version=1.9.0
-          echo $version
-          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version
+          odfe_version=$ODFE_VERSION
+          echo odfe version: $odfe_version
+          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$odfe_version
           then
-            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile
+            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$odfe_version" >> Dockerfile
             ## The ESRestTest Client uses http by default.
             ## Need to disable the security plugin to call the rest api over http.
             echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro_security ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi" >> Dockerfile
@@ -29,12 +118,11 @@ jobs:
           docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-ad:test
           sleep 90
           curl -XGET http://localhost:9200/_cat/plugins
-
       - name: Checkout Kibana
         uses: actions/checkout@v2
         with:
           repository: opendistro-for-elasticsearch/kibana-oss
-          ref: 7.8.0
+          ref: ${{ env.KIBANA_VERSION }}
           token: ${{ secrets.KIBANA_OSS_ACCESS }}
           path: kibana
       - name: Get node and yarn versions
diff --git a/.github/workflows/unit-tests-workflow.yml b/.github/workflows/unit-tests-workflow.yml
@@ -3,7 +3,8 @@ on:
   push:
     branches:
       - master
-
+env:
+  KIBANA_VERSION: 7.8.0
 jobs:
   tests:
     name: Run unit tests
@@ -16,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: opendistro-for-elasticsearch/kibana-oss
-          ref: 7.8.0
+          ref: ${{ env.KIBANA_VERSION }}
           token: ${{ secrets.KIBANA_OSS_ACCESS }}
           path: kibana
       - name: Get node and yarn versions
diff --git a/README.md b/README.md
@@ -67,6 +67,11 @@ Example output: `./build/opendistro-anomaly-detection-kibana-1.9.0.0.zip`
 
   Start Kibana, wait for it to be available on `localhost:5601`, and runs end-to-end tests.
 
+- `yarn cy:run`
+
+  Runs end-to-end tests on a currently running Kibana server. Defaults to run the tests on `localhost:5601`, although you can change this to run on any
+  Kibana server with the command `yarn cy:run --config baseUrl=<your-custom-URL>`
+
 ## Contributing to Open Distro for Elasticsearch Anomaly detection Kibana
 
 We welcome you to get involved in development, documentation, testing the anomaly detection plugin. See our [CONTRIBUTING.md](./CONTRIBUTING.md) and join in.
diff --git a/cypress.json b/cypress.json
diff --git a/package.json b/package.json
