Configure sharding for Studio e2e tests (#43211)

## Problem

The e2e test suite for Studio takes around 20min. Feedback loop is too
long

## Solution

Try enabling [playwright
sharding](https://playwright.dev/docs/test-sharding#merge-reports-cli).
Only 2 shards for now

## Results

Before:
<img width="867" height="113" alt="image"
src="https://github.com/user-attachments/assets/aef01026-0eaf-4bd0-9a56-c0f5123dd2cc"
/>
<img width="845" height="115" alt="image"
src="https://github.com/user-attachments/assets/46fcf523-8128-4cf4-83cb-3f4e414337d8"
/>

After:
<img width="716" height="105" alt="image"
src="https://github.com/user-attachments/assets/ada0ffd7-3a9f-4ca0-9b2e-98b21df3188b"
/>

## Next steps

In future dedicated PRs, improve the tests themselves.

---------

Co-authored-by: Charis <26616127+charislam@users.noreply.github.com>

.github/workflows/studio-e2e-test.yml

@@ -15,8 +15,16 @@ permissions:
 
 jobs:
   test:
+    name: 'E2E tests'
     timeout-minutes: 60
     runs-on: blacksmith-4vcpu-ubuntu-2404
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
+    outputs:
+      tests_ran: ${{ steps.filter.outputs.studio == 'true' }}
 
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -55,13 +63,65 @@ jobs:
         if: steps.filter.outputs.studio == 'true'
         run: pnpm -C e2e/studio exec playwright install chromium --with-deps --only-shell
 
+      - name: Set up NextJS/Turbo cache
+        if: steps.filter.outputs.studio == 'true'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          # See here for caching with `yarn`, `bun` or other package managers https://github.com/actions/cache/blob/main/examples.md or you can leverage caching with actions/setup-node https://github.com/actions/setup-node
+          path: |
+            .turbo/cache
+            apps/studio/.next/build
+            apps/studio/.next/cache
+          # Generate a new cache whenever packages or source files change.
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('apps/studio/**/*.js', 'apps/studio/**/*.jsx', 'apps/studio/**/*.ts', 'apps/studio/**/*.tsx') }}
+          # If source files changed but packages didn't, rebuild from a prior cache.
+          restore-keys: |
+            ${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}-
+
       - name: 🚀 Run Playwright tests against Vercel Preview
         if: steps.filter.outputs.studio == 'true'
         id: playwright
-        run: pnpm e2e
+        run: pnpm e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: always() && steps.filter.outputs.studio == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: e2e/studio/blob-report
+          retention-days: 7
+
+      - name: Fail job if tests failed
+        if: steps.filter.outputs.studio == 'true' && steps.playwright.outcome != 'success'
+        run: |
+          echo "E2E tests failed" >&2
+          exit 1
+
+  merge-reports:
+    name: 'E2E reports'
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: ${{ !cancelled() && needs.test.outputs.tests_ran == 'true' }}
+    needs: [test]
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - name: Use Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v 5.0.0
+        with:
+          path: e2e/studio/blob-report
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge Playwright reports
+        run: npx playwright merge-reports --config=e2e/studio/playwright.merge.config.ts -- e2e/studio/blob-report
 
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        if: always() && steps.filter.outputs.studio == 'true'
         with:
           name: playwright-artifacts
           path: |
@@ -70,14 +130,21 @@ jobs:
           retention-days: 7
 
       - name: Comment Playwright test results on PR
+        if: always() && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork
         uses: daun/playwright-report-comment@be9e270edd5ad86038604d3caa84a819a6ff6fed # v3.10.0
-        if: always() && steps.filter.outputs.studio == 'true' && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork
         with:
           report-file: e2e/studio/test-results/test-results.json
           comment-title: '🎭 Playwright Test Results'
 
-      - name: Fail job if tests failed
-        if: steps.filter.outputs.studio == 'true' && (steps.playwright.outcome != 'success' || steps.summarize.outputs.flaky_count > 0)
-        run: |
-          echo "E2E tests failed" >&2
-          exit 1
+  merge-results:
+    name: 'E2E results'
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: ${{ !cancelled() && needs.test.outputs.tests_ran == 'true' }}
+    steps:
+      - name: All tests ok
+        if: ${{ !(contains(needs.*.result, 'failure')) }}
+        run: exit 0
+      - name: Some tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1

e2e/studio/features/realtime-inspector.spec.ts

@@ -1,5 +1,5 @@
 import { expect } from '@playwright/test'
-import { test } from '../utils/test.js'
+
 import {
   getMessageCount,
   joinChannel,
@@ -10,6 +10,7 @@ import {
   stopListening,
   waitForRealtimeMessage,
 } from '../utils/realtime-helpers.js'
+import { test } from '../utils/test.js'
 
 const testChannelName = 'pw_realtime_test_channel'
 
@@ -116,6 +117,7 @@ test.describe('Realtime Inspector', () => {
       await openBroadcastModal(page)
 
       const codeEditor = page.getByRole('textbox', { name: /Editor content/i })
+      await expect(codeEditor).toBeInViewport({ timeout: 5000 })
       await codeEditor.click({ force: true })
       await page.keyboard.press('ControlOrMeta+KeyA')
       await page.keyboard.type('{ invalid json }')

e2e/studio/playwright.config.ts

@@ -1,4 +1,5 @@
 import { defineConfig } from '@playwright/test'
+
 import { env, STORAGE_STATE_PATH } from './env.config.js'
 
 const IS_CI = !!process.env.CI
@@ -132,10 +133,12 @@ export default defineConfig({
       },
     },
   ],
-  reporter: [
-    ['list'],
-    ['html', { open: 'never' }],
-    ['json', { outputFile: 'test-results/test-results.json' }],
-  ],
+  reporter: IS_CI
+    ? [['list'], ['blob']]
+    : [
+        ['list'],
+        ['html', { open: 'never' }],
+        ['json', { outputFile: 'test-results/test-results.json' }],
+      ],
   webServer: createWebServerConfig(),
 })

e2e/studio/playwright.merge.config.ts

@@ -0,0 +1,7 @@
+export default {
+  testDir: './features',
+  reporter: [
+    ['html', { open: 'never' }],
+    ['json', { outputFile: 'test-results/test-results.json' }],
+  ],
+}

package.json

@@ -27,7 +27,7 @@
     "test:ui-patterns": "turbo run test --filter=ui-patterns",
     "test:studio": "turbo run test --filter=studio",
     "test:studio:watch": "turbo run test --filter=studio -- watch",
-    "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
+    "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && if [ -z \"${CI}\" ]; then supabase db reset; fi && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
     "e2e:setup:selfhosted": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start",
     "e2e:setup:platform": "SKIP_ASSET_UPLOAD=1 NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && pnpm --prefix ./apps/studio start",
     "e2e": "pnpm --prefix e2e/studio run e2e",
@@ -65,6 +65,13 @@
     "pnpm": "10.24",
     "node": ">=22"
   },
-  "keywords": ["postgres", "firebase", "storage", "functions", "database", "auth"],
+  "keywords": [
+    "postgres",
+    "firebase",
+    "storage",
+    "functions",
+    "database",
+    "auth"
+  ],
   "packageManager": "pnpm@10.24.0"
 }