Skip to content

Commit e42ee72

Browse files
t3chguyt3chguy
authored
Fix more flaky playwright tests (#29007)
* Group systemic playwright flakes Signed-off-by: Michael Telatynski <[email protected]> * Fix more flaky tests Signed-off-by: Michael Telatynski <[email protected]> * Fix another flake Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * delint Signed-off-by: Michael Telatynski <[email protected]> * Fix more flakes Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Fix skip tests being wrong Signed-off-by: Michael Telatynski <[email protected]> --------- Signed-off-by: Michael Telatynski <[email protected]>
1 parent 7d30413 commit e42ee72

File tree

14 files changed

+104
-65
lines changed

14 files changed

+104
-65
lines changed

‎playwright/e2e/audio-player/audio-player.spec.ts

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,14 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
1313
import { Layout } from "../../../src/settings/enums/Layout";
1414
import { ElementAppPage } from "../../pages/ElementAppPage";
1515

16+
// Find and click "Reply" button
17+
const clickButtonReply = async (tile: Locator) => {
18+
await expect(async () => {
19+
await tile.hover();
20+
await tile.getByRole("button", { name: "Reply", exact: true }).click();
21+
}).toPass();
22+
};
23+
1624
test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
1725
test.use({
1826
displayName: "Hanako",
@@ -222,8 +230,7 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
222230

223231
// Find and click "Reply" button on MessageActionBar
224232
const tile = page.locator(".mx_EventTile_last");
225-
await tile.hover();
226-
await tile.getByRole("button", { name: "Reply", exact: true }).click();
233+
await clickButtonReply(tile);
227234

228235
// Reply to the player with another audio file
229236
await uploadFile(page, "playwright/sample-files/1sec.ogg");
@@ -251,26 +258,20 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
251258

252259
const tile = page.locator(".mx_EventTile_last");
253260

254-
// Find and click "Reply" button
255-
const clickButtonReply = async () => {
256-
await tile.hover();
257-
await tile.getByRole("button", { name: "Reply", exact: true }).click();
258-
};
259-
260261
await uploadFile(page, "playwright/sample-files/upload-first.ogg");
261262

262263
// Assert that the audio player is rendered
263264
await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible();
264265

265-
await clickButtonReply();
266+
await clickButtonReply(tile);
266267

267268
// Reply to the player with another audio file
268269
await uploadFile(page, "playwright/sample-files/upload-second.ogg");
269270

270271
// Assert that the audio player is rendered
271272
await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible();
272273

273-
await clickButtonReply();
274+
await clickButtonReply(tile);
274275

275276
// Reply to the player with yet another audio file to create a reply chain
276277
await uploadFile(page, "playwright/sample-files/upload-third.ogg");

‎playwright/e2e/knock/create-knock-room.spec.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ test.describe("Create Knock Room", () => {
8181

8282
const spotlightDialog = await app.openSpotlight();
8383
await spotlightDialog.filter(Filter.PublicRooms);
84+
await spotlightDialog.search("Cyber");
8485
await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity");
8586
});
8687
});

‎playwright/e2e/knock/knock-into-room.spec.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,7 @@ test.describe("Knock Into Room", () => {
284284

285285
const spotlightDialog = await app.openSpotlight();
286286
await spotlightDialog.filter(Filter.PublicRooms);
287+
await spotlightDialog.search("Cyber");
287288
await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity");
288289
await spotlightDialog.results.nth(0).click();
289290

‎playwright/e2e/messages/messages.spec.ts

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ async function editMessage(page: Page, message: Locator, newMsg: string): Promis
5858
await editComposer.press("Enter");
5959
}
6060

61+
const screenshotOptions = (page?: Page) => ({
62+
mask: page ? [page.locator(".mx_MessageTimestamp")] : undefined,
63+
// Hide the jump to bottom button in the timeline to avoid flakiness
64+
css: `
65+
.mx_JumpToBottomButton {
66+
display: none !important;
67+
}
68+
`,
69+
});
70+
6171
test.describe("Message rendering", () => {
6272
[
6373
{ direction: "ltr", displayName: "Quentin" },
@@ -79,24 +89,28 @@ test.describe("Message rendering", () => {
7989
await page.goto(`#/room/${room.roomId}`);
8090

8191
const msgTile = await sendMessage(page, "Hello, world!");
82-
await expect(msgTile).toMatchScreenshot(`basic-message-ltr-${direction}displayname.png`, {
83-
mask: [page.locator(".mx_MessageTimestamp")],
84-
});
92+
await expect(msgTile).toMatchScreenshot(
93+
`basic-message-ltr-${direction}displayname.png`,
94+
screenshotOptions(page),
95+
);
8596
},
8697
);
8798

8899
test("should render an LTR emote", async ({ page, user, app, room }) => {
89100
await page.goto(`#/room/${room.roomId}`);
90101

91102
const msgTile = await sendMessage(page, "/me lays an egg");
92-
await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`);
103+
await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`, screenshotOptions());
93104
});
94105

95106
test("should render an LTR rich text emote", async ({ page, user, app, room }) => {
96107
await page.goto(`#/room/${room.roomId}`);
97108

98109
const msgTile = await sendMessage(page, "/me lays a *free range* egg");
99-
await expect(msgTile).toMatchScreenshot(`emote-rich-ltr-${direction}displayname.png`);
110+
await expect(msgTile).toMatchScreenshot(
111+
`emote-rich-ltr-${direction}displayname.png`,
112+
screenshotOptions(),
113+
);
100114
});
101115

102116
test("should render an edited LTR message", async ({ page, user, app, room }) => {
@@ -106,9 +120,10 @@ test.describe("Message rendering", () => {
106120

107121
await editMessage(page, msgTile, "Hello, universe!");
108122

109-
await expect(msgTile).toMatchScreenshot(`edited-message-ltr-${direction}displayname.png`, {
110-
mask: [page.locator(".mx_MessageTimestamp")],
111-
});
123+
await expect(msgTile).toMatchScreenshot(
124+
`edited-message-ltr-${direction}displayname.png`,
125+
screenshotOptions(page),
126+
);
112127
});
113128

114129
test("should render a reply of a LTR message", async ({ page, user, app, room }) => {
@@ -122,32 +137,37 @@ test.describe("Message rendering", () => {
122137
]);
123138

124139
await replyMessage(page, msgTile, "response to multiline message");
125-
await expect(msgTile).toMatchScreenshot(`reply-message-ltr-${direction}displayname.png`, {
126-
mask: [page.locator(".mx_MessageTimestamp")],
127-
});
140+
await expect(msgTile).toMatchScreenshot(
141+
`reply-message-ltr-${direction}displayname.png`,
142+
screenshotOptions(page),
143+
);
128144
});
129145

130146
test("should render a basic RTL text message", async ({ page, user, app, room }) => {
131147
await page.goto(`#/room/${room.roomId}`);
132148

133149
const msgTile = await sendMessage(page, "مرحبا بالعالم!");
134-
await expect(msgTile).toMatchScreenshot(`basic-message-rtl-${direction}displayname.png`, {
135-
mask: [page.locator(".mx_MessageTimestamp")],
136-
});
150+
await expect(msgTile).toMatchScreenshot(
151+
`basic-message-rtl-${direction}displayname.png`,
152+
screenshotOptions(page),
153+
);
137154
});
138155

139156
test("should render an RTL emote", async ({ page, user, app, room }) => {
140157
await page.goto(`#/room/${room.roomId}`);
141158

142159
const msgTile = await sendMessage(page, "/me يضع بيضة");
143-
await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`);
160+
await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`, screenshotOptions());
144161
});
145162

146163
test("should render a richtext RTL emote", async ({ page, user, app, room }) => {
147164
await page.goto(`#/room/${room.roomId}`);
148165

149166
const msgTile = await sendMessage(page, "/me أضع بيضة *حرة النطاق*");
150-
await expect(msgTile).toMatchScreenshot(`emote-rich-rtl-${direction}displayname.png`);
167+
await expect(msgTile).toMatchScreenshot(
168+
`emote-rich-rtl-${direction}displayname.png`,
169+
screenshotOptions(),
170+
);
151171
});
152172

153173
test("should render an edited RTL message", async ({ page, user, app, room }) => {
@@ -157,9 +177,10 @@ test.describe("Message rendering", () => {
157177

158178
await editMessage(page, msgTile, "مرحبا بالكون!");
159179

160-
await expect(msgTile).toMatchScreenshot(`edited-message-rtl-${direction}displayname.png`, {
161-
mask: [page.locator(".mx_MessageTimestamp")],
162-
});
180+
await expect(msgTile).toMatchScreenshot(
181+
`edited-message-rtl-${direction}displayname.png`,
182+
screenshotOptions(page),
183+
);
163184
});
164185

165186
test("should render a reply of a RTL message", async ({ page, user, app, room }) => {
@@ -173,9 +194,10 @@ test.describe("Message rendering", () => {
173194
]);
174195

175196
await replyMessage(page, msgTile, "مرحبا بالعالم!");
176-
await expect(msgTile).toMatchScreenshot(`reply-message-trl-${direction}displayname.png`, {
177-
mask: [page.locator(".mx_MessageTimestamp")],
178-
});
197+
await expect(msgTile).toMatchScreenshot(
198+
`reply-message-trl-${direction}displayname.png`,
199+
screenshotOptions(page),
200+
);
179201
});
180202
});
181203
});

‎playwright/e2e/pinned-messages/pinned-messages.spec.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ test.describe("Pinned messages", () => {
3535
mask: [tile.locator(".mx_MessageTimestamp")],
3636
// Hide the jump to bottom button in the timeline to avoid flakiness
3737
css: `
38-
.mx_JumpToBottomButton {
39-
display: none !important;
40-
}
41-
`,
38+
.mx_JumpToBottomButton {
39+
display: none !important;
40+
}
41+
`,
4242
});
4343
},
4444
);

‎playwright/e2e/settings/encryption-user-tab/index.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,7 @@ class Helpers {
8989
await expect(dialog.getByText(title, { exact: true })).toBeVisible();
9090
await expect(dialog).toMatchScreenshot(screenshot);
9191

92-
const handle = await this.page.evaluateHandle(() => navigator.clipboard.readText());
93-
const clipboardContent = await handle.jsonValue();
92+
const clipboardContent = await this.app.getClipboard();
9493
await dialog.getByRole("textbox").fill(clipboardContent);
9594
await dialog.getByRole("button", { name: confirmButtonLabel }).click();
9695
await expect(dialog).toMatchScreenshot("default-recovery.png");

‎playwright/e2e/settings/encryption-user-tab/recovery.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ test.describe("Recovery section in Encryption tab", () => {
5353

5454
test(
5555
"should change the recovery key",
56-
{ tag: "@screenshot" },
56+
{ tag: ["@screenshot", "@no-webkit"] },
5757
async ({ page, app, homeserver, credentials, util, context }) => {
5858
await verifySession(app, "new passphrase");
5959
const dialog = await util.openEncryptionTab();
@@ -81,7 +81,7 @@ test.describe("Recovery section in Encryption tab", () => {
8181
},
8282
);
8383

84-
test("should setup the recovery key", { tag: "@screenshot" }, async ({ page, app, util }) => {
84+
test("should setup the recovery key", { tag: ["@screenshot", "@no-webkit"] }, async ({ page, app, util }) => {
8585
await verifySession(app, "new passphrase");
8686
await util.removeSecretStorageDefaultKeyId();
8787

‎playwright/e2e/spaces/spaces.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ test.describe("Spaces", () => {
8484

8585
// Copy matrix.to link
8686
await page.getByRole("button", { name: "Share invite link" }).click();
87-
expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`);
87+
expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`);
8888

8989
// Go to space home
9090
await page.getByRole("button", { name: "Go to my first room" }).click();
@@ -177,7 +177,7 @@ test.describe("Spaces", () => {
177177
const shareDialog = page.locator(".mx_SpacePublicShare");
178178
// Copy link first
179179
await shareDialog.getByRole("button", { name: "Share invite link" }).click();
180-
expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`);
180+
expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`);
181181
// Start Matrix invite flow
182182
await shareDialog.getByRole("button", { name: "Invite people" }).click();
183183

‎playwright/e2e/spaces/threads-activity-centre/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,13 @@ export const test = base.extend<{
3838
room1Name: "Room 1",
3939
room1: async ({ room1Name: name, app, user, bot }, use) => {
4040
const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] });
41+
await bot.awaitRoomMembership(roomId);
4142
await use({ name, roomId });
4243
},
4344
room2Name: "Room 2",
4445
room2: async ({ room2Name: name, app, user, bot }, use) => {
4546
const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] });
47+
await bot.awaitRoomMembership(roomId);
4648
await use({ name, roomId });
4749
},
4850
msg: async ({ page, app, util }, use) => {

‎playwright/e2e/timeline/timeline.spec.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1195,6 +1195,7 @@ test.describe("Timeline", () => {
11951195
});
11961196

11971197
await sendImage(app.client, room.roomId, NEW_AVATAR);
1198+
await app.timeline.scrollToBottom();
11981199
await expect(page.locator(".mx_MImageBody").first()).toBeVisible();
11991200

12001201
// Exclude timestamp and read marker from snapshot

‎playwright/flaky-reporter.ts

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,40 @@ type PaginationLinks = {
2424
first?: string;
2525
};
2626

27+
// We see quite a few test flakes which are caused by the app exploding
28+
// so we have some magic strings we check the logs for to better track the flake with its cause
29+
const SPECIAL_CASES = {
30+
"ChunkLoadError": "ChunkLoadError",
31+
"Unreachable code should not be executed": "Rust crypto panic",
32+
"Out of bounds memory access": "Rust crypto memory error",
33+
};
34+
2735
class FlakyReporter implements Reporter {
2836
private flakes = new Map<string, TestCase[]>();
2937

3038
public onTestEnd(test: TestCase): void {
3139
// Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
3240
if (["Dendrite", "Pinecone"].includes(test.parent.project()?.name)) return;
33-
const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`;
41+
let failures = [`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`];
3442
if (test.outcome() === "flaky") {
35-
if (!this.flakes.has(title)) {
36-
this.flakes.set(title, []);
43+
const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
44+
const pageLogs = timedOutRuns.flatMap((result) =>
45+
result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
46+
);
47+
// If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
48+
const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
49+
pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body.includes(log)),
50+
);
51+
if (specialCases.length > 0) {
52+
failures = specialCases.map((specialCase) => SPECIAL_CASES[specialCase]);
53+
}
54+
55+
for (const title of failures) {
56+
if (!this.flakes.has(title)) {
57+
this.flakes.set(title, []);
58+
}
59+
this.flakes.get(title).push(test);
3760
}
38-
this.flakes.get(title).push(test);
3961
}
4062
}
4163

‎playwright/pages/ElementAppPage.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,6 @@ export class ElementAppPage {
158158
return button.click();
159159
}
160160

161-
public async getClipboardText(): Promise<string> {
162-
return this.page.evaluate("navigator.clipboard.readText()");
163-
}
164-
165161
public async openSpotlight(): Promise<Spotlight> {
166162
const spotlight = new Spotlight(this.page);
167163
await spotlight.open();

‎playwright/pages/client.ts

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import type {
1515
ICreateRoomOpts,
1616
ISendEventResponse,
1717
MatrixClient,
18-
Room,
1918
MatrixEvent,
2019
ReceiptType,
2120
IRoomDirectoryOptions,
@@ -178,21 +177,12 @@ export class Client {
178177
*/
179178
public async createRoom(options: ICreateRoomOpts): Promise<string> {
180179
const client = await this.prepareClient();
181-
return await client.evaluate(async (cli, options) => {
180+
const roomId = await client.evaluate(async (cli, options) => {
182181
const { room_id: roomId } = await cli.createRoom(options);
183-
if (!cli.getRoom(roomId)) {
184-
await new Promise<void>((resolve) => {
185-
const onRoom = (room: Room) => {
186-
if (room.roomId === roomId) {
187-
cli.off(window.matrixcs.ClientEvent.Room, onRoom);
188-
resolve();
189-
}
190-
};
191-
cli.on(window.matrixcs.ClientEvent.Room, onRoom);
192-
});
193-
}
194182
return roomId;
195183
}, options);
184+
await this.awaitRoomMembership(roomId);
185+
return roomId;
196186
}
197187

198188
/**

‎playwright/services.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,13 @@ export const test = base.extend<TestFixtures, Services & Options>({
155155
{ scope: "worker" },
156156
],
157157

158-
context: async ({ homeserverType, synapseConfig, logger, context, request, homeserver }, use, testInfo) => {
158+
context: async (
159+
{ homeserverType, synapseConfig, logger, context, request, _homeserver, homeserver },
160+
use,
161+
testInfo,
162+
) => {
159163
testInfo.skip(
160-
!(homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0,
164+
!(_homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0,
161165
`Test specifies Synapse config options so is unsupported with ${homeserverType}`,
162166
);
163167
homeserver.setRequest(request);

0 commit comments

Comments
 (0)