Extract functions for service worker usage, and add initial MSC3916 playwright test (when supported) (#12414)

* Send user credentials to service worker for MSC3916 authentication

* appease linter

* Add initial test

The test fails, seemingly because the service worker isn't being installed or because the network mock can't reach that far.

* Remove unsafe access token code

* Split out base IDB operations to avoid importing `document` in serviceworkers

* Use safe crypto access for service workers

* Fix tests/unsafe access

* Remove backwards compatibility layer & appease linter

* Add docs

* Fix tests

* Appease the linter

* Iterate tests

* Factor out pickle key handling for service workers

* Enable everything we can about service workers

* Appease the linter

* Add docs

* Rename win32 image to linux in hopes of it just working

* Use actual image

* Apply suggestions from code review

Co-authored-by: Richard van der Hoff <[email protected]>

* Improve documentation

* Document `??` not working

* Try to appease the tests

* Add some notes

---------

Co-authored-by: Richard van der Hoff <[email protected]>

Author: turt2live

playwright/e2e/timeline/timeline.spec.ts

@@ -70,6 +70,22 @@ const sendEvent = async (client: Client, roomId: string, html = false): Promise<
     return client.sendEvent(roomId, null, "m.room.message" as EventType, content);
 };
 
+const sendImage = async (
+    client: Client,
+    roomId: string,
+    pngBytes: Buffer,
+    additionalContent?: any,
+): Promise<ISendEventResponse> => {
+    const upload = await client.uploadContent(pngBytes, { name: "image.png", type: "image/png" });
+    return client.sendEvent(roomId, null, "m.room.message" as EventType, {
+        ...(additionalContent ?? {}),
+
+        msgtype: "m.image" as MsgType,
+        body: "image.png",
+        url: upload.content_uri,
+    });
+};
+
 test.describe("Timeline", () => {
     test.use({
         displayName: OLD_NAME,
@@ -1136,5 +1152,91 @@ test.describe("Timeline", () => {
                 screenshotOptions,
             );
         });
+
+        async function testImageRendering(page: Page, app: ElementAppPage, room: { roomId: string }) {
+            await app.viewRoomById(room.roomId);
+
+            // Reinstall the service workers to clear their implicit caches (global-level stuff)
+            await page.evaluate(async () => {
+                const registrations = await window.navigator.serviceWorker.getRegistrations();
+                registrations.forEach((r) => r.update());
+            });
+
+            await sendImage(app.client, room.roomId, NEW_AVATAR);
+            await expect(page.locator(".mx_MImageBody").first()).toBeVisible();
+
+            // Exclude timestamp and read marker from snapshot
+            const screenshotOptions = {
+                mask: [page.locator(".mx_MessageTimestamp")],
+                css: `
+                    .mx_TopUnreadMessagesBar, .mx_MessagePanel_myReadMarker {
+                        display: none !important;
+                    }
+                `,
+            };
+
+            await expect(page.locator(".mx_ScrollPanel")).toMatchScreenshot(
+                "image-in-timeline-default-layout.png",
+                screenshotOptions,
+            );
+        }
+
+        test("should render images in the timeline", async ({ page, app, room, context }) => {
+            await testImageRendering(page, app, room);
+        });
+
+        // XXX: This test doesn't actually work because the service worker relies on IndexedDB, which Playwright forces
+        // to be a localstorage implementation, which service workers cannot access.
+        // See https://github.com/microsoft/playwright/issues/11164
+        // See https://github.com/microsoft/playwright/issues/15684#issuecomment-2070862042
+        //
+        // In practice, this means this test will *always* succeed because it ends up relying on fallback behaviour tested
+        // above (unless of course the above tests are also broken).
+        test.describe("MSC3916 - Authenticated Media", () => {
+            test("should render authenticated images in the timeline", async ({ page, app, room, context }) => {
+                // Note: we have to use `context` instead of `page` for routing, otherwise we'll miss Service Worker events.
+                // See https://playwright.dev/docs/service-workers-experimental#network-events-and-routing
+
+                // Install our mocks and preventative measures
+                await context.route("**/_matrix/client/versions", async (route) => {
+                    // Force enable MSC3916, which may require the service worker's internal cache to be cleared later.
+                    const json = await (await route.fetch()).json();
+                    if (!json["unstable_features"]) json["unstable_features"] = {};
+                    json["unstable_features"]["org.matrix.msc3916"] = true;
+                    await route.fulfill({ json });
+                });
+                await context.route("**/_matrix/media/*/download/**", async (route) => {
+                    // should not be called. We don't use `abort` so that it's clearer in the logs what happened.
+                    await route.fulfill({
+                        status: 500,
+                        json: { errcode: "M_UNKNOWN", error: "Unexpected route called." },
+                    });
+                });
+                await context.route("**/_matrix/media/*/thumbnail/**", async (route) => {
+                    // should not be called. We don't use `abort` so that it's clearer in the logs what happened.
+                    await route.fulfill({
+                        status: 500,
+                        json: { errcode: "M_UNKNOWN", error: "Unexpected route called." },
+                    });
+                });
+                await context.route("**/_matrix/client/unstable/org.matrix.msc3916/download/**", async (route) => {
+                    expect(route.request().headers()["Authorization"]).toBeDefined();
+                    // we can't use route.continue() because no configured homeserver supports MSC3916 yet
+                    await route.fulfill({
+                        body: NEW_AVATAR,
+                    });
+                });
+                await context.route("**/_matrix/client/unstable/org.matrix.msc3916/thumbnail/**", async (route) => {
+                    expect(route.request().headers()["Authorization"]).toBeDefined();
+                    // we can't use route.continue() because no configured homeserver supports MSC3916 yet
+                    await route.fulfill({
+                        body: NEW_AVATAR,
+                    });
+                });
+
+                // We check the same screenshot because there should be no user-visible impact to using authentication.
+                await testImageRendering(page, app, room);
+            });
+        });
     });
 });

playwright/element-web-test.ts

@@ -33,6 +33,10 @@ import { Bot, CreateBotOpts } from "./pages/bot";
 import { ProxyInstance, SlidingSyncProxy } from "./plugins/sliding-sync-proxy";
 import { Webserver } from "./plugins/webserver";
 
+// Enable experimental service worker support
+// See https://playwright.dev/docs/service-workers-experimental#how-to-enable
+process.env["PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS"] = "1";
+
 const CONFIG_JSON: Partial<IConfigOptions> = {
     // This is deliberately quite a minimal config.json, so that we can test that the default settings
     // actually work.

playwright/snapshots/timeline/timeline.spec.ts/image-in-timeline-default-layout-linux.png

@@ -34,10 +34,11 @@ import { CheckUpdatesPayload } from "./dispatcher/payloads/CheckUpdatesPayload";
 import { Action } from "./dispatcher/actions";
 import { hideToast as hideUpdateToast } from "./toasts/UpdateToast";
 import { MatrixClientPeg } from "./MatrixClientPeg";
-import { idbLoad, idbSave, idbDelete } from "./utils/StorageManager";
+import { idbLoad, idbSave, idbDelete } from "./utils/StorageAccess";
 import { ViewRoomPayload } from "./dispatcher/payloads/ViewRoomPayload";
 import { IConfigOptions } from "./IConfigOptions";
 import SdkConfig from "./SdkConfig";
+import { buildAndEncodePickleKey, getPickleAdditionalData } from "./utils/tokens/pickling";
 
 export const SSO_HOMESERVER_URL_KEY = "mx_sso_hs_url";
 export const SSO_ID_SERVER_URL_KEY = "mx_sso_is_url";
@@ -352,55 +353,21 @@ export default abstract class BasePlatform {
 
     /**
      * Get a previously stored pickle key.  The pickle key is used for
-     * encrypting libolm objects.
+     * encrypting libolm objects and react-sdk-crypto data.
      * @param {string} userId the user ID for the user that the pickle key is for.
-     * @param {string} userId the device ID that the pickle key is for.
+     * @param {string} deviceId the device ID that the pickle key is for.
      * @returns {string|null} the previously stored pickle key, or null if no
      *     pickle key has been stored.
      */
     public async getPickleKey(userId: string, deviceId: string): Promise<string | null> {
-        if (!window.crypto || !window.crypto.subtle) {
-            return null;
-        }
-        let data;
+        let data: { encrypted?: BufferSource; iv?: BufferSource; cryptoKey?: CryptoKey } | undefined;
         try {
             data = await idbLoad("pickleKey", [userId, deviceId]);
         } catch (e) {
             logger.error("idbLoad for pickleKey failed", e);
         }
-        if (!data) {
-            return null;
-        }
-        if (!data.encrypted || !data.iv || !data.cryptoKey) {
-            logger.error("Badly formatted pickle key");
-            return null;
-        }
-
-        const additionalData = this.getPickleAdditionalData(userId, deviceId);
 
-        try {
-            const key = await crypto.subtle.decrypt(
-                { name: "AES-GCM", iv: data.iv, additionalData },
-                data.cryptoKey,
-                data.encrypted,
-            );
-            return encodeUnpaddedBase64(key);
-        } catch (e) {
-            logger.error("Error decrypting pickle key");
-            return null;
-        }
-    }
-
-    private getPickleAdditionalData(userId: string, deviceId: string): Uint8Array {
-        const additionalData = new Uint8Array(userId.length + deviceId.length + 1);
-        for (let i = 0; i < userId.length; i++) {
-            additionalData[i] = userId.charCodeAt(i);
-        }
-        additionalData[userId.length] = 124; // "|"
-        for (let i = 0; i < deviceId.length; i++) {
-            additionalData[userId.length + 1 + i] = deviceId.charCodeAt(i);
-        }
-        return additionalData;
+        return (await buildAndEncodePickleKey(data, userId, deviceId)) ?? null;
     }
 
     /**
@@ -424,7 +391,7 @@ export default abstract class BasePlatform {
         const iv = new Uint8Array(32);
         crypto.getRandomValues(iv);
 
-        const additionalData = this.getPickleAdditionalData(userId, deviceId);
+        const additionalData = getPickleAdditionalData(userId, deviceId);
         const encrypted = await crypto.subtle.encrypt({ name: "AES-GCM", iv, additionalData }, cryptoKey, randomArray);
 
         try {

src/BasePlatform.ts

@@ -37,6 +37,7 @@ import ActiveWidgetStore from "./stores/ActiveWidgetStore";
 import PlatformPeg from "./PlatformPeg";
 import { sendLoginRequest } from "./Login";
 import * as StorageManager from "./utils/StorageManager";
+import * as StorageAccess from "./utils/StorageAccess";
 import SettingsStore from "./settings/SettingsStore";
 import { SettingLevel } from "./settings/SettingLevel";
 import ToastStore from "./stores/ToastStore";
@@ -493,7 +494,7 @@ export interface IStoredSession {
 async function getStoredToken(storageKey: string): Promise<string | undefined> {
     let token: string | undefined;
     try {
-        token = await StorageManager.idbLoad("account", storageKey);
+        token = await StorageAccess.idbLoad("account", storageKey);
     } catch (e) {
         logger.error(`StorageManager.idbLoad failed for account:${storageKey}`, e);
     }
@@ -502,7 +503,7 @@ async function getStoredToken(storageKey: string): Promise<string | undefined> {
         if (token) {
             try {
                 // try to migrate access token to IndexedDB if we can
-                await StorageManager.idbSave("account", storageKey, token);
+                await StorageAccess.idbSave("account", storageKey, token);
                 localStorage.removeItem(storageKey);
             } catch (e) {
                 logger.error(`migration of token ${storageKey} to IndexedDB failed`, e);
@@ -1064,7 +1065,7 @@ async function clearStorage(opts?: { deleteEverything?: boolean }): Promise<void
         AbstractLocalStorageSettingsHandler.clear();
 
         try {
-            await StorageManager.idbDelete("account", ACCESS_TOKEN_STORAGE_KEY);
+            await StorageAccess.idbDelete("account", ACCESS_TOKEN_STORAGE_KEY);
         } catch (e) {
             logger.error("idbDelete failed for account:mx_access_token", e);
         }

src/Lifecycle.ts

@@ -0,0 +1,132 @@
+/*
+Copyright 2019-2021, 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Retrieves the IndexedDB factory object.
+ *
+ * @returns {IDBFactory | undefined} The IndexedDB factory object if available, or undefined if it is not supported.
+ */
+export function getIDBFactory(): IDBFactory | undefined {
+    // IndexedDB loading is lazy for easier testing.
+
+    // just *accessing* _indexedDB throws an exception in firefox with
+    // indexeddb disabled.
+    try {
+        // `self` is preferred for service workers, which access this file's functions.
+        // We check `self` first because `window` returns something which doesn't work for service workers.
+        // Note: `self?.indexedDB ?? window.indexedDB` breaks in service workers for unknown reasons.
+        return self?.indexedDB ? self.indexedDB : window.indexedDB;
+    } catch (e) {}
+}
+
+let idb: IDBDatabase | null = null;
+
+async function idbInit(): Promise<void> {
+    if (!getIDBFactory()) {
+        throw new Error("IndexedDB not available");
+    }
+    idb = await new Promise((resolve, reject) => {
+        const request = getIDBFactory()!.open("matrix-react-sdk", 1);
+        request.onerror = reject;
+        request.onsuccess = (): void => {
+            resolve(request.result);
+        };
+        request.onupgradeneeded = (): void => {
+            const db = request.result;
+            db.createObjectStore("pickleKey");
+            db.createObjectStore("account");
+        };
+    });
+}
+
+/**
+ * Loads an item from an IndexedDB table within the underlying `matrix-react-sdk` database.
+ *
+ * If IndexedDB access is not supported in the environment, an error is thrown.
+ *
+ * @param {string} table The name of the object store in IndexedDB.
+ * @param {string | string[]} key The key where the data is stored.
+ * @returns {Promise<any>} A promise that resolves with the retrieved item from the table.
+ */
+export async function idbLoad(table: string, key: string | string[]): Promise<any> {
+    if (!idb) {
+        await idbInit();
+    }
+    return new Promise((resolve, reject) => {
+        const txn = idb!.transaction([table], "readonly");
+        txn.onerror = reject;
+
+        const objectStore = txn.objectStore(table);
+        const request = objectStore.get(key);
+        request.onerror = reject;
+        request.onsuccess = (event): void => {
+            resolve(request.result);
+        };
+    });
+}
+
+/**
+ * Saves data to an IndexedDB table within the underlying `matrix-react-sdk` database.
+ *
+ * If IndexedDB access is not supported in the environment, an error is thrown.
+ *
+ * @param {string} table The name of the object store in the IndexedDB.
+ * @param {string|string[]} key The key to use for storing the data.
+ * @param {*} data The data to be saved.
+ * @returns {Promise<void>} A promise that resolves when the data is saved successfully.
+ */
+export async function idbSave(table: string, key: string | string[], data: any): Promise<void> {
+    if (!idb) {
+        await idbInit();
+    }
+    return new Promise((resolve, reject) => {
+        const txn = idb!.transaction([table], "readwrite");
+        txn.onerror = reject;
+
+        const objectStore = txn.objectStore(table);
+        const request = objectStore.put(data, key);
+        request.onerror = reject;
+        request.onsuccess = (event): void => {
+            resolve();
+        };
+    });
+}
+
+/**
+ * Deletes a record from an IndexedDB table within the underlying `matrix-react-sdk` database.
+ *
+ * If IndexedDB access is not supported in the environment, an error is thrown.
+ *
+ * @param {string} table The name of the object store where the record is stored.
+ * @param {string|string[]} key The key of the record to be deleted.
+ * @returns {Promise<void>} A Promise that resolves when the record(s) have been successfully deleted.
+ */
+export async function idbDelete(table: string, key: string | string[]): Promise<void> {
+    if (!idb) {
+        await idbInit();
+    }
+    return new Promise((resolve, reject) => {
+        const txn = idb!.transaction([table], "readwrite");
+        txn.onerror = reject;
+
+        const objectStore = txn.objectStore(table);
+        const request = objectStore.delete(key);
+        request.onerror = reject;
+        request.onsuccess = (): void => {
+            resolve();
+        };
+    });
+}