Skip to content

Commit a7e936d

Browse files
jrichter1karthikjeeyar
jrichter1
and
karthikjeeyar
authored
test(quay): add first playwright tests (#1201)
* test(quay): add first playwright tests * Fix typo Co-authored-by: Karthik Jeeyar <[email protected]> --------- Co-authored-by: Karthik Jeeyar <[email protected]>
1 parent 352c609 commit a7e936d

File tree

8 files changed

+316
-2
lines changed

8 files changed

+316
-2
lines changed

‎.github/workflows/pr-playwright.yaml

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
name: Playwright tests
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
7+
jobs:
8+
changes:
9+
name: Scan for changes
10+
runs-on: ubuntu-latest
11+
outputs:
12+
plugins: ${{ steps.scan.outputs.plugins }}
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
fetch-depth: 0
17+
- uses: actions/setup-node@v3
18+
with:
19+
node-version: 18
20+
- name: Determine changes
21+
id: scan
22+
env:
23+
HEAD: ${{ github.event.pull_request.head.sha }}
24+
BASE: ${{ github.event.pull_request.base.sha }}
25+
run: |
26+
root=$(pwd)
27+
cd plugins
28+
29+
changed=()
30+
for f in */; do
31+
if git diff --name-only $BASE $HEAD | grep $f -q; then
32+
if [[ ! -L "$f" && -f "$f/package.json" ]]; then
33+
cd $f
34+
35+
if npm run | grep ui-test -q; then
36+
changed+=($f)
37+
fi
38+
cd $root/plugins
39+
fi
40+
fi
41+
done
42+
43+
JSON="[$(echo ${changed[@]} | sed 's/ /,/g')]"
44+
echo "plugins=$(echo $JSON)" >> $GITHUB_OUTPUT
45+
playwright:
46+
name: 'Run Playwright Tests'
47+
needs: changes
48+
if: needs.changes.outputs.plugins != '[]'
49+
runs-on: ubuntu-latest
50+
steps:
51+
- uses: actions/checkout@v4
52+
- uses: actions/setup-node@v3
53+
with:
54+
node-version: 18
55+
- name: Install dependencies
56+
run: yarn
57+
- name: Install playwright
58+
run: yarn playwright install --with-deps chromium
59+
- name: Run tests
60+
env:
61+
PLUGINS: ${{ needs.changes.outputs.plugins }}
62+
run: |
63+
root=$(pwd)
64+
cd packages/backend
65+
readarray folders < <(echo $PLUGINS | sed 's/[][]//g' | sed 's/,/ /g')
66+
67+
# Start backend
68+
echo "Starting backend"
69+
logfile=$(mktemp)
70+
yarn start >$logfile 2>&1 &
71+
72+
for attempt in $(seq 1 45); do
73+
sleep 1
74+
if grep -q "Error:" $logfile; then
75+
cat $logfile
76+
exit 1
77+
fi
78+
if grep -q "Listening on" $logfile; then
79+
echo "Backend started"
80+
break
81+
fi
82+
if [[ attempt -eq 45 ]]; then
83+
echo "Failed to launch backend"
84+
cat $logfile
85+
exit 1
86+
fi
87+
done
88+
89+
cd $root/plugins
90+
91+
# Launch suitable plugins with changes
92+
for f in $folders; do
93+
cd $f
94+
95+
echo "Starting $f plugin"
96+
tmpfile=$(mktemp)
97+
98+
# Start the plugin
99+
yarn start >$tmpfile 2>&1 &
100+
for attempt in $(seq 1 45); do
101+
sleep 1
102+
if grep -q "Error:" $tmpfile; then
103+
cat $tmpfile
104+
exit 1
105+
fi
106+
if grep -q "webpack compiled" $tmpfile; then
107+
echo "$f started"
108+
break
109+
fi
110+
if [[ attempt -eq 45 ]]; then
111+
echo "Failed to launch $f"
112+
cat $tmpfile
113+
exit 1
114+
fi
115+
done
116+
117+
# Run UI tests
118+
yarn run ui-test
119+
120+
# Kill the plugin
121+
pid=$(lsof -i :3000 -Fp | grep p | sed s/p//)
122+
kill -9 $pid && echo "$f shut down"
123+
cd $root/plugins
124+
done
125+
126+
# Kill backend
127+
pid=$(lsof -i :7007 -Fp | grep p | sed s/p//)
128+
kill -9 $pid && echo "Backend shut down"
129+
- uses: actions/upload-artifact@v3
130+
if: always()
131+
with:
132+
name: playwright-report
133+
path: plugins/*/playwright-report/
134+
retention-days: 1

‎.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,9 @@ site
6565
.webpack-cache
6666

6767
.tmp
68+
69+
# playwright
70+
test-results/
71+
playwright-report/
72+
blob-report/
73+
playwright/.cache/

‎CONTRIBUTING.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,45 @@ Before pushing your code changes make sure all **tests pass** and the **coverage
312312
$ yarn test
313313
```
314314

315+
### UI Tests
316+
317+
Some plugins (e.g. [quay](https://github.com/janus-idp/backstage-plugins/tree/main/plugins/quay)) also have playwright-based UI tests. When making changes to such plugin, make sure these tests pass.
318+
319+
To run the UI tests locally, take the following steps:
320+
321+
First, install playwright dependencies:
322+
323+
```bash
324+
$ yarn install --with-deps chromium
325+
```
326+
327+
The remaining steps need to be run in parallel.
328+
Launch the backend package and wait for it to start:
329+
330+
```bash
331+
$ cd packages/backend && yarn start
332+
```
333+
334+
Launch the plugin:
335+
336+
```bash
337+
$ cd plugins/${plugin} && yarn start
338+
```
339+
340+
Finally, launch the UI tests (headless):
341+
342+
```bash
343+
$ cd plugins/${plugin} && yarn run ui-test
344+
```
345+
346+
If you wish to see the test runner UI, instead of headless:
347+
348+
```bash
349+
$ cd plugins/${plugin} && yarn playwright test --ui
350+
```
351+
352+
Test results from the headless run will be available in `plugins/${plugin}/playwright-report` folder.
353+
315354
## Releasing changes
316355

317356
This repository defaults to a rapid release scheme where we would rather release on every PR merge than restrict ourselves by a strict release cadence and policy. This brings contributors the opportunity to see the direct impact of their contributions since they are released immediately after the merge. The release process itself is done via the [semantic-release](https://semantic-release.gitbook.io/semantic-release/) tool. In order for it to work properly, it requires contributors to follow a simple set of rules:

‎plugins/quay/dev/index.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const mockEntity: Entity = {
1313
name: 'backstage',
1414
description: 'backstage.io',
1515
annotations: {
16-
'quay.io/repository-slug': 'janus-idp/redhat-backstage-build',
16+
'quay.io/repository-slug': 'backstage-test/test-images',
1717
},
1818
},
1919
spec: {

‎plugins/quay/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"prepare": "",
2424
"start": "backstage-cli package start",
2525
"test": "backstage-cli package test --passWithNoTests --coverage",
26-
"tsc": "tsc"
26+
"tsc": "tsc",
27+
"ui-test": "yarn playwright test"
2728
},
2829
"dependencies": {
2930
"@backstage/catalog-model": "^1.4.3",
@@ -47,6 +48,7 @@
4748
"@backstage/dev-utils": "1.0.22",
4849
"@backstage/test-utils": "1.4.4",
4950
"@janus-idp/cli": "1.7.1",
51+
"@playwright/test": "^1.41.0",
5052
"@testing-library/jest-dom": "5.17.0",
5153
"@testing-library/react": "12.1.5",
5254
"@testing-library/react-hooks": "8.0.1",

‎plugins/quay/playwright.config.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { defineConfig, devices } from '@playwright/test';
2+
3+
/**
4+
* See https://playwright.dev/docs/test-configuration.
5+
*/
6+
export default defineConfig({
7+
testDir: './tests',
8+
/* Run tests in files in parallel */
9+
fullyParallel: true,
10+
/* Fail the build on CI if you accidentally left test.only in the source code. */
11+
forbidOnly: !!process.env.CI,
12+
/* Retry on CI only */
13+
retries: process.env.CI ? 2 : 0,
14+
/* Run tests in sequence. */
15+
workers: 1,
16+
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
17+
reporter: 'html',
18+
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
19+
use: {
20+
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
21+
trace: 'on-first-retry',
22+
screenshot: 'only-on-failure',
23+
video: 'retain-on-failure',
24+
},
25+
26+
/* Configure projects for major browsers */
27+
projects: [
28+
{
29+
name: 'chromium',
30+
use: { ...devices['Desktop Chrome'] },
31+
},
32+
],
33+
});

‎plugins/quay/tests/quay.spec.ts

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { expect, Page, test } from '@playwright/test';
2+
3+
test.describe('Quay plugin', () => {
4+
let page: Page;
5+
6+
test.beforeAll(async ({ browser }) => {
7+
const context = await browser.newContext();
8+
page = await context.newPage();
9+
await page.goto('http://localhost:3000/quay');
10+
await expect(
11+
page.getByRole('link', { name: 'backstage-test/test-images' }),
12+
).toBeEnabled({ timeout: 20000 });
13+
});
14+
15+
test.afterAll(async ({ browser }) => {
16+
await browser.close();
17+
});
18+
19+
test('All columns are shown', async () => {
20+
const columns = [
21+
'Tag',
22+
'Last Modified',
23+
'Security Scan',
24+
'Size',
25+
'Expires',
26+
'Manifest',
27+
];
28+
const thead = page.locator('thead');
29+
30+
for (const col of columns) {
31+
await expect(thead.getByText(col)).toBeVisible();
32+
}
33+
});
34+
35+
test('Vulnerabilities are listed', async () => {
36+
const severity = ['High:', 'Medium:', 'Low:'];
37+
for (const lvl of severity) {
38+
await expect(page.getByRole('link', { name: lvl })).toBeVisible();
39+
}
40+
});
41+
42+
test('Vulnerability details are accessible', async () => {
43+
await page.getByRole('link', { name: 'High' }).first().click();
44+
await expect(page.getByText('Vulnerabilities for')).toBeVisible({
45+
timeout: 15000,
46+
});
47+
});
48+
49+
test('Vulnerability columns are shown', async () => {
50+
const columns = [
51+
'Advisory',
52+
'Severity',
53+
'Package Name',
54+
'Current Version',
55+
'Fixed By',
56+
];
57+
58+
for (const col of columns) {
59+
await expect(page.getByText(col)).toBeVisible();
60+
}
61+
});
62+
63+
test('Vulnerability rows are shown', async () => {
64+
const tbody = page.locator('tbody');
65+
await expect(tbody.locator('tr')).toHaveCount(5);
66+
});
67+
68+
test('Link back to repository works', async () => {
69+
await page.getByRole('link', { name: 'Back to repository' }).click();
70+
await expect(
71+
page.getByRole('link', { name: 'backstage-test/test-images' }),
72+
).toBeEnabled();
73+
});
74+
});

‎yarn.lock

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9543,6 +9543,13 @@
95439543
resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
95449544
integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
95459545

9546+
"@playwright/test@^1.41.0":
9547+
version "1.41.2"
9548+
resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.2.tgz#bd9db40177f8fd442e16e14e0389d23751cdfc54"
9549+
integrity sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg==
9550+
dependencies:
9551+
playwright "1.41.2"
9552+
95469553
"@pmmmwh/react-refresh-webpack-plugin@^0.5.5", "@pmmmwh/react-refresh-webpack-plugin@^0.5.7":
95479554
version "0.5.11"
95489555
resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz#7c2268cedaa0644d677e8c4f377bc8fb304f714a"
@@ -21535,6 +21542,11 @@ fs.realpath@^1.0.0:
2153521542
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
2153621543
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
2153721544

21545+
[email protected]:
21546+
version "2.3.2"
21547+
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
21548+
integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
21549+
2153821550
fsevents@^2.3.2, fsevents@~2.3.2:
2153921551
version "2.3.3"
2154021552
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -29400,6 +29412,20 @@ [email protected], pkginfo@^0.4.1:
2940029412
resolved "https://registry.yarnpkg.com/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff"
2940129413
integrity sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ==
2940229414

29415+
[email protected]:
29416+
version "1.41.2"
29417+
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.2.tgz#db22372c708926c697acc261f0ef8406606802d9"
29418+
integrity sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA==
29419+
29420+
[email protected]:
29421+
version "1.41.2"
29422+
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.2.tgz#4e760b1c79f33d9129a8c65cc27953be6dd35042"
29423+
integrity sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A==
29424+
dependencies:
29425+
playwright-core "1.41.2"
29426+
optionalDependencies:
29427+
fsevents "2.3.2"
29428+
2940329429
pluralize@^8.0.0:
2940429430
version "8.0.0"
2940529431
resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"

0 commit comments

Comments
 (0)