[JS] Add ability to assert against visual results, Add upload retry (#152)

Co-authored-by: Logan Graham <[email protected]>

Author: omacranger

visual-js/.changeset/modern-poets-hang.md

@@ -0,0 +1,10 @@
+---
+"@saucelabs/visual-playwright": minor
+"@saucelabs/visual-storybook": minor
+"@saucelabs/cypress-visual-plugin": minor
+"@saucelabs/wdio-sauce-visual-service": minor
+"@saucelabs/visual": minor
+---
+
+Unify result checking behavior into utility
+Add automatic retry mechanism to file uploads for timeouts

visual-js/visual-cypress/Dockerfile

@@ -2,22 +2,14 @@ FROM cypress/included:latest AS runner
 
 WORKDIR app
 
-COPY package.json .
+RUN corepack enable
 
-COPY ./visual-cypress/src ./visual-cypress/src
-COPY ./visual-cypress/tsconfig.json ./visual-cypress/tsconfig.json
-COPY ./visual-cypress/package.json ./visual-cypress/package.json
+COPY . ./
 
-RUN npm install --workspace=visual-cypress
-RUN npm run build --workspace=visual-cypress
+RUN yarn install && npm run build --workspaces --if-present
 
-COPY ./visual-cypress/integration-tests/cypress ./integration-tests/cypress
-COPY ./visual-cypress/integration-tests/cypress.config.js ./integration-tests/cypress.config.js
-COPY ./visual-cypress/integration-tests/package.json ./integration-tests/package.json
-COPY ./visual-cypress/integration-tests/tsconfig.json ./integration-tests/tsconfig.json
-
-WORKDIR integration-tests
+WORKDIR ./visual-cypress/integration-tests
 
 RUN npm install
 
-ENTRYPOINT ["npm", "run", "login-test"]
+ENTRYPOINT ["npm", "run", "login-test"]

visual-js/visual-cypress/src/commands.ts

@@ -16,6 +16,7 @@ import {
   VisualCheckOptions,
   VisualRegion,
 } from './types';
+import type { DiffStatus } from '@saucelabs/visual';
 
 declare global {
   namespace Cypress {
@@ -33,7 +34,7 @@ declare global {
         options?: VisualCheckOptions,
       ): Chainable<Subject>;
 
-      sauceVisualResults(): Chainable<Subject>;
+      sauceVisualResults(): Chainable<Record<DiffStatus, number>>;
     }
     interface EndToEndConfigOptions {
       saucelabs: SauceConfig;

visual-js/visual-cypress/src/index.ts

@@ -19,6 +19,7 @@ import {
   selectiveRegionOptionsToDiffingOptions,
   VisualApi,
   VisualApiRegion,
+  getVisualResults,
 } from '@saucelabs/visual';
 import {
   HasSauceConfig,
@@ -265,35 +266,10 @@ Sauce Labs Visual: Unable to create new build.
   }
 
   private async getResultSummary(): Promise<Record<DiffStatus, number>> {
-    const diffsForTestResult = await this.api.diffsForTestResult(this.buildId!);
-    if (!diffsForTestResult) {
-      throw new Error('Something went wrong while fetching test results');
-    }
-
-    const filterDiffsById = (diff: { id: string; status: DiffStatus }) =>
-      this.uploadedDiffIds.includes(diff.id);
-    const initialStatusSummary = {
-      [DiffStatus.Queued]: 0,
-      [DiffStatus.Unapproved]: 0,
-      [DiffStatus.Approved]: 0,
-      [DiffStatus.Equal]: 0,
-      [DiffStatus.Errored]: 0,
-      [DiffStatus.Rejected]: 0,
-    } satisfies Record<DiffStatus, number>;
-    const statusSummary = diffsForTestResult.nodes
-      .filter(filterDiffsById)
-      .reduce((statusSummary, diff) => {
-        if (!statusSummary[diff.status]) {
-          statusSummary[diff.status] = 0;
-        }
-        statusSummary[diff.status]++;
-        return statusSummary;
-      }, initialStatusSummary);
-    if (statusSummary[DiffStatus.Queued]) {
-      throw new DiffNotReadyError('Some diffs are not ready');
-    }
-
-    return statusSummary;
+    return await getVisualResults(this.api, {
+      buildId: this.buildId,
+      diffIds: this.uploadedDiffIds,
+    });
   }
 
   async getTestResults(): Promise<Record<DiffStatus, number>> {

visual-js/visual-playwright/package.json

@@ -33,10 +33,12 @@
     "test": "jest --collect-coverage"
   },
   "dependencies": {
-    "@playwright/test": "^1.42.1",
     "@saucelabs/visual": "^0.9.0",
     "exponential-backoff": "^3.1.1"
   },
+  "peerDependencies": {
+    "@playwright/test": "^1.42.1"
+  },
   "tsup": {
     "entry": [
       "./src/index.ts"
@@ -54,6 +56,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^28.0.0 || ^29.0.0",
+    "@playwright/test": "^1.48.0",
     "@storybook/types": "^8.0.2",
     "@tsconfig/node18": "^2.0.0",
     "@types/node": "^18.13.0",

visual-js/visual-playwright/src/api.ts

@@ -4,6 +4,7 @@ import {
   downloadDomScript,
   getApi as getVisualApi,
   getDomScript,
+  getVisualResults,
   RegionIn,
   removeDomScriptFile,
   VisualEnvOpts,
@@ -16,6 +17,7 @@ const clientVersion = 'PKG_VERSION';
 
 export class VisualPlaywright {
   constructor(public client: string = `visual-playwright/${clientVersion}`) {}
+  uploadedDiffIds: Record<string, string[]> = {};
 
   public get api() {
     let api = globalThis.visualApi;
@@ -135,11 +137,12 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       testName: string | undefined;
       suiteName: string | undefined;
       deviceName: string | undefined;
+      testId: string;
     },
     name: string,
     options?: Partial<SauceVisualParams>,
   ) {
-    const { testName, suiteName, deviceName } = info;
+    const { testName, suiteName, deviceName, testId } = info;
     const { buildId } = getOpts();
 
     if (!buildId) {
@@ -156,7 +159,13 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       ignoreRegions: userIgnoreRegions,
       diffingMethod,
     } = options ?? {};
-    const { animations = 'disabled', caret } = screenshotOptions;
+    const {
+      animations = 'disabled',
+      caret,
+      fullPage = true,
+      style,
+      timeout,
+    } = screenshotOptions;
     let ignoreRegions: RegionIn[] = [];
 
     const promises: Promise<unknown>[] = [
@@ -270,7 +279,9 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
     const devicePixelRatio = await page.evaluate(() => window.devicePixelRatio);
 
     const screenshotBuffer = await page.screenshot({
-      fullPage: true,
+      fullPage,
+      style,
+      timeout,
       animations,
       caret,
       clip,
@@ -318,12 +329,24 @@ ${e instanceof Error ? e.message : JSON.stringify(e)}
       diffingMethod,
     });
 
-    await this.api.createSnapshot({
+    const { diffs } = await this.api.createSnapshot({
       ...meta,
       testName,
       suiteName,
       uploadUuid: uploadId,
     });
+
+    if (!this.uploadedDiffIds[testId]) this.uploadedDiffIds[testId] = [];
+
+    this.uploadedDiffIds[testId].push(...diffs.nodes.map((diff) => diff.id));
+  }
+
+  public async visualResults({ testId }: { testId: string }) {
+    const { buildId } = getOpts();
+    return await getVisualResults(this.api, {
+      buildId,
+      diffIds: this.uploadedDiffIds[testId] ?? [],
+    });
   }
 
   /**

visual-js/visual-playwright/src/fixtures.ts

@@ -1,11 +1,23 @@
-import { TestFixture, TestInfo } from '@playwright/test';
-import { Page } from 'playwright-core';
-import { SauceVisualParams } from './types';
-import { sauceVisualCheck } from './playwright';
+import type { TestFixture, TestInfo } from '@playwright/test';
+import type { Page } from 'playwright-core';
+import type { SauceVisualParams } from './types';
+import { sauceVisualCheck, sauceVisualResults } from './playwright';
+import type { DiffStatus } from '@saucelabs/visual';
 
 export type SauceVisualFixtures = {
   sauceVisual: {
+    /**
+     * Takes a snapshot of the current page and uploads it to Sauce Labs for visual diffing.
+     * @param name
+     * @param options
+     */
     visualCheck: (name: string, options?: SauceVisualParams) => Promise<void>;
+    /**
+     * Returns the visual results for the active test. Can only be used inside a
+     * `test` or `afterEach` block since it uses the current test context for finding matching
+     * visual results.
+     */
+    sauceVisualResults: () => Promise<Record<DiffStatus, number>>;
   };
 };
 
@@ -20,6 +32,7 @@ export const sauceVisualFixtures: (defaultOptions?: SauceVisualParams) => {
           ...options,
         });
       },
+      sauceVisualResults: async () => await sauceVisualResults(testInfo),
     });
   },
 });

visual-js/visual-playwright/src/index.ts

@@ -2,6 +2,7 @@ export {
   sauceVisualSetup,
   sauceVisualTeardown,
   sauceVisualCheck,
+  sauceVisualResults,
 } from './playwright';
 export { sauceVisualFixtures, SauceVisualFixtures } from './fixtures';
 export { SauceVisualParams } from './types';

visual-js/visual-playwright/src/playwright.ts

@@ -1,15 +1,19 @@
 import VisualPlaywright from './api';
 import type { Page } from 'playwright-core';
-import { TestInfo } from '@playwright/test';
-import { SauceVisualParams } from './types';
-import { VisualEnvOpts } from '@saucelabs/visual';
+import type { TestInfo } from '@playwright/test';
+import type { SauceVisualParams } from './types';
+import type { VisualEnvOpts } from '@saucelabs/visual';
 
 export const sauceVisualSetup = async (opts?: Partial<VisualEnvOpts>) => {
   VisualPlaywright.globalSetup(opts);
   return VisualPlaywright.setup();
 };
 export const sauceVisualTeardown =
   VisualPlaywright.teardown.bind(VisualPlaywright);
+
+/**
+ * Takes a snapshot of the current page and uploads it to Sauce Labs for visual diffing.
+ */
 export const sauceVisualCheck = async (
   page: Page,
   testInfo: TestInfo,
@@ -22,7 +26,18 @@ export const sauceVisualCheck = async (
       deviceName: testInfo.project.name,
       testName: testInfo.title,
       suiteName: testInfo.titlePath.slice(0, -1).join('/'),
+      testId: testInfo.testId,
     },
     name,
     options,
   );
+
+/**
+ * Returns the visual results for the active test. Can only be used inside a
+ * `test` or `afterEach` block since it uses the current test context for finding matching
+ * visual results.
+ */
+export const sauceVisualResults = async (testInfo: TestInfo) =>
+  await VisualPlaywright.visualResults({
+    testId: testInfo.testId,
+  });

visual-js/visual-playwright/src/types.ts

@@ -2,7 +2,10 @@ import { DiffingMethod, RegionIn, VisualEnvOpts } from '@saucelabs/visual';
 import { PageScreenshotOptions } from 'playwright-core';
 
 export interface SauceVisualParams {
-  screenshotOptions?: Pick<PageScreenshotOptions, 'animations' | 'caret'>;
+  screenshotOptions?: Pick<
+    PageScreenshotOptions,
+    'animations' | 'caret' | 'fullPage' | 'style' | 'timeout'
+  >;
   /**
    * Whether we should capture a dom snapshot.
    */

visual-js/visual-storybook/src/api.ts

@@ -33,6 +33,7 @@ export const postVisit = async (page: Page, context: TestContext) => {
       testName: undefined,
       suiteName: undefined,
       deviceName: deviceName || undefined,
+      testId: context.id,
     },
     `${context.title}/${context.name}`,
     {

visual-js/visual-wdio/package.json

@@ -56,7 +56,7 @@
     "build": "tsc --declaration -p tsconfig.build.json",
     "watch": "tsc -w --declaration -p tsconfig.build.json",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
-    "test": "jest --collect-coverage"
+    "test": "jest --collect-coverage --passWithNoTests"
   },
   "publishConfig": {
     "access": "public"