Skip to content

Commit d856544

Browse files
ofekatrofekatr
authored
Merge pull request #3838 from snyk/chore/create-iac-e2e-tests-cfg-2106
chore: Create smoke tests for experimental IaC tests
2 parents 60f7a48 + 8890fb3 commit d856544

File tree

6 files changed

+231
-1
lines changed

6 files changed

+231
-1
lines changed

‎.github/CODEOWNERS

+3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ test/fixtures/container-app-vulns/ @snyk/mycelium
3434
test/fixtures/container-projects/ @snyk/mycelium @snyk/potion
3535
test/fixtures/docker/ @snyk/mycelium @snyk/potion
3636
test/fixtures/iac/ @snyk/group-infrastructure-as-code
37+
test/smoke/iac/ @snyk/group-infrastructure-as-code
3738
test/smoke/spec/iac/ @snyk/group-infrastructure-as-code
3839
test/smoke/spec/snyk_code_spec.sh @snyk/zenith
3940
test/smoke/spec/snyk_basic_spec.sh @snyk/hammer
@@ -70,6 +71,8 @@ test/fixtures/unmanaged-log4j-fixture @snyk/tundra
7071
test/jest/acceptance/snyk-log4shell/log4shell-detection.spec.ts @snyk/tundra
7172
test/jest/acceptance/snyk-test/app-vuln-container-project.spec.ts @snyk/mycelium
7273
/.github @snyk/hammer
74+
/.github/workflows/iac-smoke-tests.yml @snyk/group-infrastructure-as-code
75+
/.github/workflows/iac-smoke-tests-pulls.yml @snyk/group-infrastructure-as-code
7376

7477
# tap tests ownership
7578
test/tap/cli-monitor/ @snyk/snyk-open-source

‎.github/workflows/iac-smoke-tests-pulls.yml

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Infrastructure as Code Smoke Tests (Pull Requests)
2+
3+
on:
4+
pull_request:
5+
branches: [master]
6+
7+
jobs:
8+
check_for_changed_iac_files:
9+
name: Check for changed IaC files
10+
runs-on: ubuntu-latest
11+
outputs:
12+
is_changed: ${{ steps.check_iac_files_changed.outputs.is_changed }}
13+
14+
steps:
15+
- uses: actions/checkout@v2
16+
17+
- name: Install jq
18+
run: |
19+
sudo apt-get install jq
20+
21+
- name: Parse CODEOWNERS file
22+
id: codeowners
23+
uses: SvanBoxel/[email protected]
24+
with:
25+
file_match_info: 'true'
26+
path: ./.github/CODEOWNERS
27+
28+
- name: Get changed files
29+
id: changed-files
30+
uses: tj-actions/[email protected]
31+
32+
- name: Get all IaC files
33+
id: get_all_iac_files
34+
run: |
35+
ALL_IAC_FILES=$(
36+
echo ${{ toJSON(steps.codeowners.outputs.filematches) }} |
37+
jq '[
38+
to_entries[] |
39+
select(
40+
.value.owners |
41+
index("@snyk/group-infrastructure-as-code")
42+
) |
43+
.key
44+
]'
45+
)
46+
47+
echo "::set-output name=all_iac_files::$(
48+
echo $ALL_IAC_FILES
49+
)"
50+
51+
- id: check_iac_files_changed
52+
name: Check for changed files owned by IaC
53+
run: |
54+
ALL_IAC_FILES=${{ toJson(steps.get_all_iac_files.outputs.all_iac_files) }}
55+
56+
CHANGED_FILES=$(
57+
echo ${{ steps.changed-files.outputs.all_changed_files }} |
58+
jq -R 'split(" ")'
59+
)
60+
61+
CHANGED_IAC_FILES=$(
62+
echo $CHANGED_FILES |
63+
jq --argjson ALL_IAC_FILES "$ALL_IAC_FILES" '[
64+
.[] |
65+
. as $changed_file |
66+
select(
67+
$ALL_IAC_FILES |
68+
index($changed_file)
69+
)
70+
]'
71+
)
72+
73+
CHANGED_IAC_FILES_COUNT=$(
74+
echo $CHANGED_IAC_FILES | jq 'length'
75+
)
76+
77+
IS_CHANGED=$(
78+
echo $CHANGED_IAC_FILES_COUNT | jq '. > 0'
79+
)
80+
81+
$IS_CHANGED &&
82+
echo "Found $CHANGED_IAC_FILES_COUNT changed IaC files: $CHANGED_IAC_FILES"||
83+
echo "No changed IaC files found!"
84+
85+
echo "::set-output name=is_changed::$IS_CHANGED"
86+
87+
run_iac_smoke_tests:
88+
name: Run IaC smoke tests
89+
uses: ./.github/workflows/iac-smoke-tests.yml
90+
needs: check_for_changed_iac_files
91+
if: ${{ needs.check_for_changed_iac_files.outputs.is_changed == 'true' }}
92+
secrets: inherit

‎.github/workflows/iac-smoke-tests.yml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Infrastructure as Code Smoke Tests
2+
3+
on:
4+
schedule:
5+
- cron: '0 * * * *'
6+
release:
7+
types: [published]
8+
workflow_call:
9+
10+
jobs:
11+
run_iac_e2e_tests:
12+
runs-on: ${{ matrix.os }}-latest
13+
strategy:
14+
fail-fast: false
15+
matrix:
16+
os: [ubuntu, macos, windows]
17+
18+
steps:
19+
- uses: actions/checkout@v2
20+
with:
21+
ref: ${{ github.ref }}
22+
23+
- uses: actions/setup-node@v3
24+
with:
25+
node-version: 15
26+
27+
- name: Install jq on macOS
28+
if: ${{ matrix.os == 'macos' }}
29+
run: |
30+
brew install jq
31+
32+
- name: Install jq on Windows
33+
if: ${{ matrix.os == 'windows'}}
34+
run: |
35+
iwr -useb get.scoop.sh -outfile 'install-scoop.ps1'
36+
.\install-scoop.ps1 -RunAsAdmin
37+
scoop install jq
38+
39+
- name: Install jq on Ubuntu
40+
if: ${{ matrix.os == 'ubuntu' }}
41+
run: |
42+
sudo apt-get install jq
43+
44+
- name: Install dependencies
45+
run: |
46+
npm install
47+
48+
- name: Build Snyk CLI
49+
run: |
50+
npm run build
51+
52+
- name: Run IaC smoke tests - non-Windows
53+
if: ${{ matrix.os != 'windows' }}
54+
env:
55+
IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
56+
TEST_SNYK_COMMAND: ${{ format('node {0}/dist/cli/index.js', github.workspace) }}
57+
run: |
58+
npm run test:smoke:iac
59+
60+
- name: Run IaC smoke tests - Windows
61+
if: ${{ matrix.os == 'windows' }}
62+
shell: pwsh
63+
env:
64+
IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
65+
TEST_SNYK_COMMAND: ${{ format('node {0}\dist\cli\index.js', github.workspace) }}
66+
run: |
67+
npm run test:smoke:iac

‎package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
"test:unit": "jest --runInBand --testPathPattern '/test(/jest)?/unit/'",
4444
"test:acceptance": "jest --runInBand --testPathPattern '/test(/jest)?/acceptance/'",
4545
"test:tap": "tap -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register test/tap/*.test.* ",
46-
"test:smoke": "./scripts/run-smoke-tests-locally.sh"
46+
"test:smoke": "./scripts/run-smoke-tests-locally.sh",
47+
"test:smoke:iac": "jest --runInBand --testPathPattern '/test/smoke(/jest)?/iac/'"
4748
},
4849
"keywords": [
4950
"security",

‎test/smoke/iac/README.md

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Snyk Infrastructure as Code Smoke Tests
2+
3+
Design goal is to have a single test suite, aligned with the scope of the Snyk CLI's smoke tests, that can detect if IaC commands do not work properly - before and after it's released. The tests help us incorporate resources and requests made via network calls, to provide better coverage for the end to end flow of these commands. Some examples:
4+
5+
- Network calls made for fetching org properties, such as feature flags data, custom severities, etc.
6+
- Downloading resources from CDNs, e.g., binary executables, ruleset bundles, etc.
7+
8+
The tests were written with Jest, and use a Snyk CLI executable either configured in the PATH environment variable, or overrode, using the `TEST_SNYK_COMMAND` environment variable, see more in the 'Notes on the
9+
10+
# Implementation details and usage
11+
12+
These smoke tests are written with Jest, using the Snyk CLI executable identified on the runtime environment (See 'Notes on the local run' section below to read on how to override it)
13+
14+
Spec in this folder is used as a
15+
16+
1. **"name: Infrastructure as Code Smoke Tests" Github Action** - these run every hour and upon releases.
17+
2. **["Infrastructure as Code Smoke Tests (Pull Requests)"] GitHub Action** - these run for pull requests to the `master` branch which include changes to files owned by group IaC.
18+
19+
```sh
20+
npm run test:smoke:iac
21+
```
22+
23+
### Notes on the local run
24+
25+
These tests can be executed with the following npm script:
26+
27+
```
28+
npm run test:smoke:iac
29+
```
30+
31+
Alternatively, they can be executed directly via `jest`, by running:
32+
33+
```
34+
npx jest test/smoke/iac/
35+
```
36+
37+
You may specify any executable that will be used by the smoke tests, by configuring the `TEST_SNYK_COMMAND` environment variable. E.g. a local exuctable `TEST_SNYK_COMMAND="./snyk-macos"` or an `TEST_SNYK_COMMAND="npx [email protected]"` or `TEST_SNYK_COMMAND="node ./dist/cli"` for local execution.
38+
39+
You may also configure an authentication token with the `SNYK_TOKEN` environment variable, to run the tests with any org and user needed.

‎test/smoke/iac/test.spec.ts

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { run } from '../../jest/acceptance/iac/helpers';
2+
3+
jest.setTimeout(1_000 * 90);
4+
5+
describe('snyk iac test --experimental', () => {
6+
beforeAll(async () => {
7+
await login();
8+
});
9+
10+
it('runs successfully and resolves with a non-error exit code', async () => {
11+
// Arrange
12+
const filePath = 'iac/depth_detection/root.tf';
13+
14+
// Act
15+
const { stderr, stdout, exitCode } = await run(
16+
`snyk iac test --experimental ${filePath}`,
17+
);
18+
19+
// Assert
20+
expect(stdout).toContain('Infrastructure as Code');
21+
expect(stderr).toBe('');
22+
expect(exitCode).toBeLessThan(2);
23+
});
24+
25+
async function login() {
26+
await run(`snyk auth ${process.env.IAC_SMOKE_TESTS_SNYK_TOKEN}`);
27+
}
28+
});

0 commit comments

Comments
 (0)