Skip to content

Introduce code coverage collection to Cypress runs #9084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
297 changes: 297 additions & 0 deletions .github/workflows/cypress_tests_workflow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,297 @@

name: Cypress 12 experimental tests

# trigger on every commit push and PR for all branches except pushes for backport branches
on:
push:
branches: ['main', '[0-9]+\.x', '[0-9]+\.[0-9]+'] # Run the functional test on push for only release branches
paths-ignore:
- '**/*.md'
- 'docs/**'
- '.lycheeignore'
- 'CODEOWNERS'
- 'changelogs/fragments/**'
pull_request:
branches: ['**']
paths-ignore:
- '**/*.md'
- 'docs/**'
- '.lycheeignore'
- 'CODEOWNERS'
- 'changelogs/fragments/**'

env:
CYPRESS_BROWSER: 'chromium'
CYPRESS_username: 'admin'
NODE_OPTIONS: '--max-old-space-size=6144 --dns-result-order=ipv4first'
COVERAGE: true
PROJECT_ARTIFACT_NAME: 'project-artifact'
COMBINATIONS: |
[
[],
["SECURITY"],
["WORKSPACE", "DATA_SOURCE", "QUERY_ENHANCEMENTS", "USE_NEW_HOME_PAGE"]
]

jobs:
build-and-validate:
name: Build and validate
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
registry-url: 'https://registry.npmjs.org'

- name: Setup Yarn
run: |
npm uninstall -g yarn
npm i -g [email protected]
yarn config set network-timeout 1000000 -g

- name: Run bootstrap
run: yarn osd bootstrap

- name: Check for yarn.lock changes
run: |
if [[ `git status --porcelain yarn.lock` ]]; then
echo -e "\033[31mThe yarn.lock file is out of sync!\033[0m"
git diff
exit 1
fi

- name: Generate dev docs
run: yarn docs:generateDevDocs

- name: Check for dev docs changes
run: |
if [[ `git status --porcelain docs/_sidebar.md` ]]; then
echo -e "\033[31mThe dev docs are out of sync; run yarn docs:generateDevDocs and amend the PR.\033[0m"
git diff
exit 1
fi

- name: Run linter
id: linter
run: yarn lint

- name: Validate NOTICE file
id: notice-validate
run: yarn notice:validate

- name: Validate licenses
id: i18n-licenses
run: yarn checkLicenses

- name: Check i18n
id: i18n-check
run: yarn i18n:check

- name: Build plugins
run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12

# upload-artifact looses permissions; we need to tar it to maintain them
- name: Archive content
run: tar -czf /tmp/osd-build.tar.gz --exclude=.git --exclude=.yarn .

- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: ${{ env.PROJECT_ARTIFACT_NAME }}
path: /tmp/osd-build.tar.gz
retention-days: 1

prepare-cypress-matrix:
name: Prepare Cypress tests
needs: build-and-validate
runs-on: ubuntu-latest
outputs:
test-matrix: ${{ steps.create-list.outputs.list }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Trigger tests for combinations
id: create-list
run: |
declare -a inputs_array

process_spec() {
local spec_path="$1"
local spec_name="$2"
local counter=1

while IFS= read -r combo; do
echo "Preparing tests for $spec_name with combination $counter: $combo"

input_object=$(jq -n \
--arg spec_pattern "$spec_path" \
--arg run_flag "${spec_name}:${counter}" \
--argjson combo "$combo" \
'{
spec_pattern: $spec_pattern,
run_flag: $run_flag
} + ($combo | to_entries | map({key: .value, value: "true"}) | from_entries)')

echo "Generated input_object:"
echo "$input_object" | jq .

inputs_array+=("$input_object")
echo "Current size of inputs_array: ${#inputs_array[@]}"

counter=$((counter + 1))
done < <(echo "$COMBINATIONS" | jq -c '.[]')
}

# Process spec files at the root
root_specs=$(find ./cypress/integration -maxdepth 1 -name "*.spec.js")
if [ -n "$root_specs" ]; then
echo "Found root specs: $root_specs"
process_spec "cypress/integration/*.spec.js" "root"
fi

# Process spec folders
for folder in ./cypress/integration/*/; do
if [ -d "$folder" ]; then
folder_name=$(basename "$folder")
echo "Processing folder: $folder_name"
folder_specs=$(find "$folder" -name "*.spec.js")
if [ -n "$folder_specs" ]; then
echo "Found specs in $folder_name: $folder_specs"
process_spec "cypress/integration/$folder_name/**/*.spec.js" "$folder_name"
fi
fi
done

if [ ${#inputs_array[@]} -eq 0 ]; then
echo "No test configurations generated!"
json_output='{"include":[]}'
else
json_output=$(printf '%s\n' "${inputs_array[@]}" | jq -s '{"include": .}')
fi

# Escape the JSON string for GitHub Actions
escaped_json_output=$(echo "$json_output" | jq -c -r @json)
echo "list=$escaped_json_output" >> $GITHUB_OUTPUT


run-tests:
name: Run tests (cypress:${{ matrix.run_flag }})
needs: [prepare-cypress-matrix, build-and-validate]
runs-on: ubuntu-latest
strategy:
matrix: ${{ fromJson(needs.prepare-cypress-matrix.outputs.test-matrix) }}
fail-fast: false
steps:
- name: Display run details
run: |
echo "Flag: ${{ matrix.run_flag }}"
echo "Spec pattern: ${{ matrix.spec_pattern }}"
echo "Security enabled: ${{ matrix.SECURITY || 'false' }}"
echo "Workspace enabled: ${{ matrix.WORKSPACE || 'false' }}"
echo "Data Source enabled: ${{ matrix.DATA_SOURCE || 'false' }}"
echo "Query Enhancements enabled: ${{ matrix.QUERY_ENHANCEMENTS || 'false' }}"
echo "New Home enabled: ${{ matrix.USE_NEW_HOME_PAGE || 'false' }}"

- name: Checkout code
uses: actions/checkout@v4

- name: Download built artifact
uses: actions/download-artifact@v4
with:
name: ${{ env.PROJECT_ARTIFACT_NAME }}
path: .

- name: Extract archive
run: |
tar -xzf osd-build.tar.gz --overwrite --ignore-command-error
rm osd-build.tar.gz

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
registry-url: 'https://registry.npmjs.org'

- name: Setup Yarn
run: |
npm uninstall -g yarn
npm i -g [email protected]
yarn config set network-timeout 1000000 -g

- name: Setup JDK
uses: actions/setup-java@v4
with:
java-version: '21'
distribution: 'corretto'

- name: Get package version
run: |
echo "OSD_VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV

- name: Generate random password
run: |
CYPRESS_password=$(openssl rand -base64 20)
echo "CYPRESS_password=$CYPRESS_password" >> $GITHUB_ENV

- name: Setup OpenSearch and Dashboards
run: |
npm i osd-launcher -g

if [ "${{ matrix.SECURITY }}" = "true" ]; then
osd-launcher -os ${{ env.OSD_VERSION }} -osd . -p ${{ env.CYPRESS_password }} --no-plugins --no-build --destination ~/
else
osd-launcher -os ${{ env.OSD_VERSION }} -osd . --no-security --no-plugins --no-build --destination ~/
fi

- name: Run OpenSearch
run: |
# Launch and forget
/bin/bash -c ~/OpenSearch-v${{ env.OSD_VERSION }}/bin/opensearch &

- name: Run OSD
run: |
WORKSPACE_FLAG=$([ "${{ matrix.WORKSPACE }}" = "true" ] && echo "--workspace.enabled=true" || echo "--workspace.enabled=false")
DATA_SOURCE_FLAG=$([ "${{ matrix.DATA_SOURCE }}" = "true" ] && echo "--data_source.enabled=true" || echo "--data_source.enabled=false")
QUERY_ENHANCEMENTS_FLAG=$([ "${{ matrix.QUERY_ENHANCEMENTS }}" = "true" ] && echo "--uiSettings.overrides["query:enhancements:enabled"]=true" || echo "--uiSettings.overrides["query:enhancements:enabled"]=false")
USE_NEW_HOME_PAGE_FLAG=$([ "${{ matrix.USE_NEW_HOME_PAGE }}" = "true" ] && echo "--uiSettings.overrides["home:useNewHomePage"]=true" || echo "--uiSettings.overrides["home:useNewHomePage"]=false")

# Launch and forget
/bin/bash -c "node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true $WORKSPACE_FLAG $DATA_SOURCE_FLAG $QUERY_ENHANCEMENTS_FLAG $USE_NEW_HOME_PAGE_FLAG" &

- name: Wait for 15 seconds
run: sleep 15s
shell: bash

- name: Run tests
run: yarn test:cypress --spec "${{ matrix.spec_pattern }}"

- name: Upload coverage
id: upload-code-coverage
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./coverage
flags: "cypress:${{ matrix.run_flag }}"

- name: Generate safe artifact name
if: always()
id: safe-name
run: |
value=$(echo "${{ matrix.run_flag }}" | sed -E 's/[^a-z0-9_-]+/_/gi' | sed -E 's/^_|_$//g')
echo "value=${value}" >> $GITHUB_OUTPUT

- name: Upload test artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: cypress_artifact_${{ steps.safe-name.outputs.value }}
path: |
cypress/screenshots
cypress/videos
coverage
retention-days: 1
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can code coverage work with split test buckets? We currently got all of our query-enhancements test split into 5 buckets due to how long they take

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes.

9 changes: 9 additions & 0 deletions changelogs/fragments/9084.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
test:
- Introduce code coverage collection to Cypress runs ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))
- Enable Cypress tests to detect features of the OSD instance they are running against ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))
- Turn on test isolation when security is disabled ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))
- Add auto-login to Cypress tests when security is enabled ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))

infra:
- Add coverage reporting to Cypress CI ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))
- Add comprehensive testing based on features during CI ([#9084](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9084))
21 changes: 16 additions & 5 deletions cypress.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
*/

import { defineConfig } from 'cypress';
import codeCoverageTask from '@cypress/code-coverage/task';
import webpackPreprocessor from '@cypress/webpack-preprocessor';
import { setupDynamicConfig } from './cypress/scripts/dynamic_config';

module.exports = defineConfig({
defaultCommandTimeout: 60000,
Expand All @@ -15,11 +17,11 @@ module.exports = defineConfig({
env: {
ENGINE: {
name: 'default',
url: 'http://localhost:9200',
url: undefined,
},
SECONDARY_ENGINE: {
name: 'test_cluster',
url: 'http://localhost:9200',
url: undefined,
},
S3_ENGINE: {
name: 'BasicS3Connection',
Expand All @@ -28,7 +30,6 @@ module.exports = defineConfig({
password: process.env.S3_CONNECTION_PASSWORD,
},
openSearchUrl: 'http://localhost:9200',
SECURITY_ENABLED: false,
AGGREGATION_VIEW: false,
username: 'admin',
password: 'myStrongPassword123!',
Expand All @@ -39,6 +40,9 @@ module.exports = defineConfig({
ML_COMMONS_DASHBOARDS_ENABLED: true,
WAIT_FOR_LOADER_BUFFER_MS: 0,
DISABLE_LOCAL_CLUSTER: false,

// This value is automatically determined at runtime
SECURITY_ENABLED: false,
},
e2e: {
baseUrl: 'http://localhost:5601',
Expand All @@ -48,10 +52,15 @@ module.exports = defineConfig({
},
});

function setupNodeEvents(
async function setupNodeEvents(
on: Cypress.PluginEvents,
config: Cypress.PluginConfigOptions
): Cypress.PluginConfigOptions {
): Promise<Cypress.PluginConfigOptions> {
if (process.env.COVERAGE) {
config.env.codeCoverage = { url: '/__coverage__' };
codeCoverageTask(on, config);
}

const { webpackOptions } = webpackPreprocessor.defaultOptions;

/**
Expand All @@ -76,5 +85,7 @@ function setupNodeEvents(
})
);

await setupDynamicConfig(config);

return config;
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { SECONDARY_ENGINE } from '../../../../../utils/constants';
const randomString = Math.random().toString(36).substring(7);
const workspace = `${WORKSPACE_NAME}-${randomString}`;

describe('No Index Pattern Check Test', () => {
ifEnabled(['WORKSPACE', '!SECURITY']).describe('No Index Pattern Check Test', () => {
before(() => {
// Load test data
cy.setupTestData(
Expand Down
Loading
Loading