test(e2e): add toContainAStory assertion to visitStory(new) (#16582)

* test(playwright): add toContainAStory assertion to visitStory

* fix: datatable, errorboundary, modal

* fix: renamed story urls for textinput, tag and uishell

* fix: fixed fluid datepicker test and skipped datatable

* fix: changed playwright config

* feat: removed commented test

* fix(playwright): re-enable tests

* test(playwright): fix flaky tests, update playwright, improve ci artifacts

* test(datepicker): fix flaky tests

* chore: yarn dedupe

* test(ci): fix playwright merge reports, flaky menu test

---------

Co-authored-by: Taylor Jones <[email protected]>
Co-authored-by: Taylor Jones <[email protected]>
Co-authored-by: Guilherme Datilio Ribeiro <[email protected]>

Author: 3 people

.github/workflows/ci.yml

@@ -68,7 +68,9 @@ jobs:
           path: |
             node_modules
             */**/node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'packages/**/yarn.lock') }}
+          key:
+            ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock',
+            'packages/**/yarn.lock') }}
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache
       - name: Build project
@@ -95,7 +97,9 @@ jobs:
           path: |
             node_modules
             */**/node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'packages/**/yarn.lock') }}
+          key:
+            ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock',
+            'packages/**/yarn.lock') }}
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache
       - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
@@ -119,8 +123,10 @@ jobs:
 
   vrt-runner:
     strategy:
+      fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -135,7 +141,9 @@ jobs:
           path: |
             node_modules
             */**/node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'packages/**/yarn.lock') }}
+          key:
+            ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock',
+            'packages/**/yarn.lock') }}
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache
       - name: Install browsers
@@ -161,15 +169,16 @@ jobs:
           PERCY_TOKEN: c9a21a3fde4fda0a0f822d633426ab26e2ab2c1cba55221d342d4047744c8c24
           PERCY_PARALLEL_TOTAL: 4
         run: |
-          yarn percy exec --parallel -- yarn playwright test --project chromium --grep @vrt --shard="${{ matrix.shard }}/4"
+          yarn percy exec --parallel -- yarn playwright test --project chromium --grep @vrt --shard="${{ matrix.shardIndex }}/${{ matrix.shardTotal }}"
       - name: Stop storybook
         run: kill ${{ steps.storybook.outputs.pid }}
-      - name: Upload test results
-        if: always()
-        uses: actions/upload-artifact@v3
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a #4.3.6
         with:
-          name: playwright-vrt-report
-          path: .playwright
+          name: playwright-vrt-blob-report-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 1
 
   vrt:
     if: ${{ always() }}
@@ -182,8 +191,10 @@ jobs:
 
   avt-runner:
     strategy:
+      fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -198,7 +209,9 @@ jobs:
           path: |
             node_modules
             */**/node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'packages/**/yarn.lock') }}
+          key:
+            ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock',
+            'packages/**/yarn.lock') }}
       - name: Install dependencies
         run: yarn install --immutable --immutable-cache
       - name: Install browsers
@@ -221,15 +234,16 @@ jobs:
       - name: Run AVT
         if: github.repository == 'carbon-design-system/carbon'
         run: |
-          yarn playwright test --project chromium --grep @avt --shard="${{ matrix.shard }}/4"
+          yarn playwright test --project chromium --grep @avt --shard="${{ matrix.shardIndex }}/${{ matrix.shardTotal }}"
       - name: Stop storybook
         run: kill ${{ steps.storybook.outputs.pid }}
-      - name: Upload test results
-        if: always()
-        uses: actions/upload-artifact@v3
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a #4.3.6
         with:
-          name: playwright-avt-report
-          path: .playwright
+          name: playwright-avt-blob-report-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 1
 
   avt:
     if: ${{ always() }}
@@ -238,4 +252,67 @@ jobs:
     steps:
       - name: Check AVT Runner job status
         if: ${{ needs.avt-runner.result != 'success' }}
-        run: exit 1
+        run: exit 1
+
+  merge-playwright-reports:
+    # Merge reports after *-runner jobs, even if some shards have failed
+    if: ${{ !cancelled() }}
+    needs: [vrt-runner, avt-runner]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b #v4.0.3
+        with:
+          node-version: '20.x'
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
+        if: github.event_name != 'merge_group'
+        id: cache
+        with:
+          path: |
+            node_modules
+            */**/node_modules
+          key:
+            ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock',
+            'packages/**/yarn.lock') }}
+      - name: Install dependencies
+        run: yarn install --immutable --immutable-cache
+
+      - name: Download vrt blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-playwright-vrt-blob-reports
+          pattern: playwright-vrt-blob-report-*
+          merge-multiple: true
+
+      - name: Merge vrt reports into HTML Report
+        run:
+          npx playwright merge-reports --reporter html
+          ./all-playwright-vrt-blob-reports
+
+      - name: Upload vrt HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-playwright-vrt-report--attempt-${{ github.run_attempt }}
+          path: playwright-vrt-report
+          retention-days: 14
+
+      - name: Download avt blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-playwright-avt-blob-reports
+          pattern: playwright-avt-blob-report-*
+          merge-multiple: true
+
+      - name: Merge avt reports into HTML Report
+        run:
+          npx playwright merge-reports --reporter html
+          ./all-playwright-avt-blob-reports
+
+      - name: Upload avt HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-playwright-avt-report--attempt-${{ github.run_attempt }}
+          path: playwright-avt-report
+          retention-days: 14

.github/workflows/release.yml

@@ -58,7 +58,7 @@ jobs:
         run: kill ${{ steps.storybook.outputs.pid }}
       - name: Upload test results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a #4.3.6
         with:
           name: playwright-avt-report
           path: .playwright

.gitignore

@@ -57,6 +57,7 @@ package-lock.json
 
 # Playwright
 .playwright
+blob-report
 
 # Local token
 .env

.yarn/cache/@percy-cli-app-npm-1.27.0-b3f1e3b4b2-53225aa1a2.zip renamed to .yarn/cache/@percy-cli-app-npm-1.29.2-96e08847c3-701dc58fe0.zip

@@ -150,7 +150,7 @@ test.describe('@avt Button', () => {
       },
     });
     await expect(page.getByRole('button')).toBeVisible();
-    page.getByRole('button').hover();
+    await page.getByRole('button').hover();
 
     await expect(page).toHaveNoACViolations('Button-hover');
   });

.yarn/cache/@percy-cli-build-npm-1.27.0-1d1385ddf1-623855a9ab.zip renamed to .yarn/cache/@percy-cli-build-npm-1.29.2-c797ed1c57-596618d7d8.zip

@@ -32,7 +32,7 @@ test.describe('@avt CodeSnippet', () => {
     });
 
     await expect(page.getByRole('button')).toBeVisible();
-    page.getByRole('button').hover();
+    await page.getByRole('button').hover();
     await expect(page).toHaveNoACViolations('CodeSnippet inline hover');
   });
 

.yarn/cache/@percy-cli-command-npm-1.27.0-c972c6e7fd-82a434fe5b.zip renamed to .yarn/cache/@percy-cli-command-npm-1.29.2-1939756e70-ff9a5ab09a.zip

@@ -276,16 +276,19 @@ test.describe('@avt DataTable', () => {
         'components-datatable-selection--default'
       );
     });
-    test('@avt-advanced-states with-radio-expansion', async ({ page }) => {
+    test.skip('@avt-advanced-states with-radio-expansion', async ({ page }) => {
       await visitStory(page, {
         component: 'DataTable',
-        id: 'components-datatable-selection--with-radio-expansion',
+        id: 'components-datatable-dynamic--playground',
         globals: {
           theme: 'white',
         },
+        args: {
+          radio: true,
+        },
       });
       await expect(page).toHaveNoACViolations(
-        'components-datatable-selection--with-radio-expansion'
+        'components-datatable-with-radio-expansion'
       );
     });
     test('@avt-advanced-states with-selection-and-sorting', async ({

.yarn/cache/@percy-cli-config-npm-1.27.0-e05ea63b52-b444f8ac56.zip renamed to .yarn/cache/@percy-cli-config-npm-1.29.2-9c9a36ac88-9cb7c08014.zip

@@ -49,7 +49,6 @@ test.describe('@avt DatePicker', () => {
     await expect(page).toHaveNoACViolations('DatePicker-Disabled');
   });
 
-  // skipping for now due to accessibility violation
   test('@avt-advanced-states skeleton', async ({ page }) => {
     await visitStory(page, {
       component: 'DatePicker',
@@ -62,8 +61,8 @@ test.describe('@avt DatePicker', () => {
     await expect(page).toHaveNoACViolations('DatePicker-Skeleton');
   });
 
-  // skipping for now due to accessibility violation
-  test('@avt-advanced-states open', async ({ page }) => {
+  // skipped due to text contrast accessibility violation
+  test.skip('@avt-advanced-states open', async ({ page }) => {
     await visitStory(page, {
       component: 'DatePicker',
       id: 'components-datepicker--playground',
@@ -92,13 +91,26 @@ test.describe('@avt DatePicker', () => {
     await expect(
       page.getByRole('textbox', { name: 'Date Picker label' })
     ).toBeFocused();
-    await expect(page.locator('div.flatpickr-calendar')).toHaveClass(/open/);
-    await page.keyboard.press('ArrowDown');
-    await expect(page.locator('span.today')).toBeFocused();
-    await page.keyboard.press('Escape');
-    await expect(page.locator('div.flatpickr-calendar')).not.toHaveClass(
-      /open/
-    );
+    const calendar = await page.locator('div.flatpickr-calendar');
+    await expect(calendar).toHaveClass(/open/);
+
+    // avoid flaky test failures from the keyboard press happening too quickly
+    // this retries the keypress along with the focus assertion until it passes
+    await expect(async () => {
+      await page.keyboard.press('ArrowDown');
+      const today = await page.locator('span.today');
+      await expect(today).toBeVisible();
+      await expect(today).toBeFocused();
+    }).toPass();
+
+    // avoid flaky test failures from the keyboard press happening too quickly
+    // this retries the keypress along with the focus assertion until it passes
+    await expect(async () => {
+      await page.keyboard.press('Escape');
+      await expect(page.locator('div.flatpickr-calendar')).not.toHaveClass(
+        /open/
+      );
+    }).toPass();
   });
 
   test('@avt-keyboard-nav range state', async ({ page }) => {

.yarn/cache/@percy-cli-exec-npm-1.27.0-85d128b976-3bcabce315.zip

@@ -25,7 +25,7 @@ test.describe('@avt ErrorBoundary', () => {
   test('@avt-advanced-states with Custom context', async ({ page }) => {
     await visitStory(page, {
       component: 'ErrorBoundary',
-      id: 'components-errorboundary--skeleton',
+      id: 'components-errorboundary--with-custom-context',
       globals: {
         theme: 'white',
       },

.yarn/cache/@percy-cli-exec-npm-1.29.2-066ce40f33-45e4a18286.zip

@@ -65,14 +65,29 @@ test.describe('@avt FluidDatePicker', () => {
     });
     // for some reason the firs tab is not working to focus the first tabstop, so focusing manually
     await page.getByRole('textbox', { name: 'Label' }).focus();
-    await expect(page.getByRole('textbox', { name: 'Label' })).toBeFocused();
-    await expect(page.locator('div.flatpickr-calendar')).toHaveClass(/open/);
-    await page.keyboard.press('ArrowDown');
-    await expect(page.locator('span.today')).toBeFocused();
-    await page.keyboard.press('Escape');
-    await expect(page.locator('div.flatpickr-calendar')).not.toHaveClass(
-      /open/
-    );
+    const textbox = await page.getByRole('textbox', { name: 'Label' });
+    await expect(textbox).toBeFocused();
+    const calendar = await page.locator('div.flatpickr-calendar');
+    await expect(calendar).toHaveClass(/open/);
+    await expect(calendar).toBeVisible();
+
+    const today = await page.locator('.flatpickr-day.today');
+
+    // avoid flaky test failures from the keyboard press happening too quickly
+    // this retries the keypress along with the focus assertion until it passes
+    await expect(async () => {
+      await page.keyboard.press('ArrowDown');
+      await expect(today).toBeVisible();
+      await expect(today).toBeFocused();
+    }).toPass();
+
+    // avoid flaky test failures from the keyboard press happening too quickly
+    // this retries the keypress along with the focus assertion until it passes
+    await expect(async () => {
+      await page.keyboard.press('Escape');
+      await expect(calendar).not.toHaveClass(/open/);
+      await expect(calendar).toBeHidden();
+    }).toPass();
   });
 
   test('@avt-keyboard-nav range', async ({ page }) => {

.yarn/cache/@percy-cli-npm-1.27.0-21cadb4899-5d6334e9ae.zip

@@ -25,7 +25,7 @@ test.describe('@avt FluidTextInput', () => {
   test('@avt-advanced-states password input', async ({ page }) => {
     await visitStory(page, {
       component: 'FluidTextInput',
-      id: 'experimental-fluid-components-unstable-fluidtextinput--password-input',
+      id: 'experimental-fluid-components-unstable-fluidpasswordinput--default',
       globals: {
         theme: 'white',
       },

.yarn/cache/@percy-cli-npm-1.29.2-2cfbaa255c-073d993182.zip

@@ -22,8 +22,8 @@ test.describe('FluidTextInput', () => {
         });
 
         await snapshotStory(page, {
-          component: 'FluidTextInput',
-          id: 'experimental-fluid-components-unstable-fluidtextinput--password-input',
+          component: 'FluidPasswordInput',
+          id: 'experimental-fluid-components-unstable-fluidpasswordinput--default',
           theme,
         });
       });