Skip to content

Commit 5448de5

Browse files
t3chguyt3chguy
authored
Run Playwright tests on Firefox & "Safari" nightly (#28757)
* Run Playwright tests on Firefox Signed-off-by: Michael Telatynski <[email protected]> * Update playwright.config.ts * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Update end-to-end-tests.yaml * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Finalise Signed-off-by: Michael Telatynski <[email protected]> * Documentation Signed-off-by: Michael Telatynski <[email protected]> * typo Signed-off-by: Michael Telatynski <[email protected]> --------- Signed-off-by: Michael Telatynski <[email protected]>
1 parent be181d2 commit 5448de5

23 files changed

+295
-190
lines changed

‎.github/workflows/end-to-end-tests.yaml

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
# as an artifact and run end-to-end tests.
44
name: End to End Tests
55
on:
6+
# CRON to run all Projects at 6am UTC
7+
schedule:
8+
- cron: "0 6 * * *"
69
pull_request: {}
710
merge_group:
811
types: [checks_requested]
@@ -32,6 +35,8 @@ concurrency:
3235
env:
3336
# fetchdep.sh needs to know our PR number
3437
PR_NUMBER: ${{ github.event.pull_request.number }}
38+
# Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total)
39+
NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }}
3540

3641
permissions: {} # No permissions required
3742

@@ -40,6 +45,9 @@ jobs:
4045
name: "Build Element-Web"
4146
runs-on: ubuntu-24.04
4247
if: inputs.skip != true
48+
outputs:
49+
num-runners: ${{ env.NUM_RUNNERS }}
50+
runners-matrix: ${{ steps.runner-vars.outputs.matrix }}
4351
steps:
4452
- name: Checkout code
4553
uses: actions/checkout@v4
@@ -79,8 +87,17 @@ jobs:
7987
path: webapp
8088
retention-days: 1
8189

90+
- name: Calculate runner variables
91+
id: runner-vars
92+
uses: actions/github-script@v7
93+
with:
94+
script: |
95+
const numRunners = parseInt(process.env.NUM_RUNNERS, 10);
96+
const matrix = Array.from({ length: numRunners }, (_, i) => i + 1);
97+
core.setOutput("matrix", JSON.stringify(matrix));
98+
8299
playwright:
83-
name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}"
100+
name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
84101
needs: build
85102
if: inputs.skip != true
86103
runs-on: ubuntu-24.04
@@ -92,7 +109,19 @@ jobs:
92109
fail-fast: false
93110
matrix:
94111
# Run multiple instances in parallel to speed up the tests
95-
runner: [1, 2, 3, 4, 5, 6]
112+
runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }}
113+
project:
114+
- Chrome
115+
- Firefox
116+
- WebKit
117+
isCron:
118+
- ${{ github.event_name == 'schedule' }}
119+
# Skip the Firefox & Safari runs unless this was a cron trigger
120+
exclude:
121+
- isCron: false
122+
project: Firefox
123+
- isCron: false
124+
project: WebKit
96125
steps:
97126
- uses: actions/checkout@v4
98127
with:
@@ -124,24 +153,30 @@ jobs:
124153
with:
125154
path: |
126155
~/.cache/ms-playwright
127-
key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-chromium
156+
key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}
128157

129-
- name: Install Playwright browser
158+
- name: Install Playwright browsers
130159
if: steps.playwright-cache.outputs.cache-hit != 'true'
131-
run: yarn playwright install --with-deps --no-shell chromium
160+
run: yarn playwright install --with-deps --no-shell
161+
162+
- name: Install system dependencies for WebKit
163+
# Some WebKit dependencies seem to lay outside the cache and will need to be installed separately
164+
if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true'
165+
run: yarn playwright install-deps webkit
132166

133167
# We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
134168
- name: Run Playwright tests
135169
run: |
136170
yarn playwright test \
137-
--shard "${{ matrix.runner }}/${{ strategy.job-total }}" \
171+
--shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
172+
--project="${{ matrix.project }}" \
138173
${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
139174
140175
- name: Upload blob report to GitHub Actions Artifacts
141176
if: always()
142177
uses: actions/upload-artifact@v4
143178
with:
144-
name: all-blob-reports-${{ matrix.runner }}
179+
name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }}
145180
path: blob-report
146181
retention-days: 1
147182

‎docs/playwright.md

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,11 @@ yarn run test:playwright:open --headed --debug
5353

5454
See more command line options at <https://playwright.dev/docs/test-cli>.
5555

56-
### Running with Rust cryptography
56+
## Projects
5757

58-
`matrix-js-sdk` is currently in the
59-
[process](https://github.com/vector-im/element-web/issues/21972) of being
60-
updated to replace its end-to-end encryption implementation to use the [Matrix
61-
Rust SDK](https://github.com/matrix-org/matrix-rust-sdk). This is not currently
62-
enabled by default, but it is possible to have Playwright configure Element to use
63-
the Rust crypto implementation by passing `--project="Rust Crypto"` or using
64-
the top left options in open mode.
58+
By default, Playwright will run all "Projects", this means tests will run against Chrome, Firefox and "Safari" (Webkit).
59+
We only run tests against Chrome in pull request CI, but all projects in the merge queue.
60+
Some tests are excluded from running on certain browsers due to incompatibilities in the test harness.
6561

6662
## How the Tests Work
6763

@@ -224,3 +220,14 @@ We use test tags to categorise tests for running subsets more efficiently.
224220

225221
- `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue.
226222
- `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection.
223+
- `@no-$project`: Tests which are unsupported in $Project. These tests will be skipped when running in $Project.
224+
225+
Anything testing Matrix media will need to have `@no-firefox` and `@no-webkit` as those rely on the service worker which
226+
has to be disabled in Playwright on Firefox & Webkit to retain routing functionality.
227+
Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
228+
there at this time.
229+
230+
## Colima
231+
232+
If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path
233+
within `$HOME` to allow bind mounting temporary directories into the Docker containers.

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
"test:playwright:open": "yarn test:playwright --ui",
6565
"test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run",
6666
"test:playwright:screenshots:build": "docker build playwright -t element-web-playwright",
67-
"test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot",
67+
"test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome",
6868
"coverage": "yarn test --coverage",
6969
"analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts",
7070
"analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp",

‎playwright.config.ts

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,49 @@ import { defineConfig, devices } from "@playwright/test";
1111
const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
1212

1313
export default defineConfig({
14-
projects: [{ name: "Chrome", use: { ...devices["Desktop Chrome"], channel: "chromium" } }],
14+
projects: [
15+
{
16+
name: "Chrome",
17+
use: {
18+
...devices["Desktop Chrome"],
19+
channel: "chromium",
20+
permissions: ["clipboard-write", "clipboard-read", "microphone"],
21+
launchOptions: {
22+
args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
23+
},
24+
},
25+
},
26+
{
27+
name: "Firefox",
28+
use: {
29+
...devices["Desktop Firefox"],
30+
launchOptions: {
31+
firefoxUserPrefs: {
32+
"permissions.default.microphone": 1,
33+
},
34+
},
35+
// This is needed to work around an issue between Playwright routes, Firefox, and Service workers
36+
// https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120
37+
serviceWorkers: "block",
38+
},
39+
ignoreSnapshots: true,
40+
},
41+
{
42+
name: "WebKit",
43+
use: {
44+
...devices["Desktop Safari"],
45+
// Seemingly WebKit has the same issue as Firefox in Playwright routes not working
46+
// https://playwright.dev/docs/network#missing-network-events-and-service-workers
47+
serviceWorkers: "block",
48+
},
49+
ignoreSnapshots: true,
50+
},
51+
],
1552
use: {
1653
viewport: { width: 1280, height: 720 },
1754
ignoreHTTPSErrors: true,
1855
video: "retain-on-failure",
1956
baseURL,
20-
permissions: ["clipboard-write", "clipboard-read", "microphone"],
21-
launchOptions: {
22-
args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
23-
},
2457
trace: "on-first-retry",
2558
},
2659
webServer: {

‎playwright/e2e/audio-player/audio-player.spec.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
1313
import { Layout } from "../../../src/settings/enums/Layout";
1414
import { ElementAppPage } from "../../pages/ElementAppPage";
1515

16-
test.describe("Audio player", () => {
16+
test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
1717
test.use({
1818
displayName: "Hanako",
1919
});

0 commit comments

Comments
 (0)