feat: Playwright tests (#21083)

* initial commit for testing the github action.

* Improvements to the Github action.

* Add docker caching.

* Fix an issue in the github action.

* Another fix for the action.

* Just delete the db after the tests.

* Add a supabase app to the playwright-tests.

* Delete .env.testing.

* Remove the docker image caching from the Playwright github action.

* Add a README.md.

* Add an example test for the table editor.

* Ignore the generated keys.json.

* Add commands for running and writing tests.

* Remove Auth tests.

* Only show the react-query tool when not in test mode.

* Add data-testids to the buttons and use them in the test.

* Handle a case where SUPABASE_URL isn't defined.

* Fix the button test id.

* Remove some dependency.

* Try setting a timeout for an action.

* Add timeout to another action.

* Add some timeouts before the save button.

* Add some timeout before clicking save.

* Enable the video recording only during local testing.

* Minor fixes.

* Expand the README.
This commit is contained in:
Ivan Vasilov
2024-02-19 12:12:54 +01:00
committed by GitHub
parent 29e447b153
commit 66ae36e005
16 changed files with 521 additions and 19 deletions
+47
View File
@@ -0,0 +1,47 @@
name: Playwright Tests
on:
push:
branches: [master]
paths:
- 'apps/studio/**'
- 'package-lock.json'
pull_request:
branches: [master]
paths:
- 'apps/studio/**'
- 'package-lock.json'
# Cancel old builds on new commit for same workflow + branch/PR
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18.x]
steps:
- uses: actions/checkout@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'
- uses: supabase/setup-cli@v1
- name: Install dependencies
run: npm ci
- name: Install Playwright Browsers
run: npx playwright install --with-deps
- name: Run Playwright tests
run: npm run test:playwright
# - uses: actions/upload-artifact@v3
# if: always()
# with:
# name: playwright-report
# path: playwright-report/
# retention-days: 30
@@ -131,6 +131,7 @@ const DefaultHeader = ({
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
data-testid="table-editor-insert-new-row"
type="primary"
size="tiny"
icon={<IconChevronDown size={14} strokeWidth={1.5} />}
@@ -57,6 +57,7 @@ const ActionBar = ({
<Button
disabled={loading || disableApply}
loading={loading}
data-testid="action-bar-save-row"
htmlType="submit"
form={formId}
>
+4 -2
View File
@@ -9,8 +9,10 @@ const path = require('path')
const API_URL = process.env.NEXT_PUBLIC_API_URL
? new URL(process.env.NEXT_PUBLIC_API_URL).origin
: ''
const SUPABASE_URL = new URL(process.env.SUPABASE_URL).origin
const GOTRUE_URL = new URL(process.env.NEXT_PUBLIC_GOTRUE_URL).origin
const SUPABASE_URL = process.env.SUPABASE_URL ? new URL(process.env.SUPABASE_URL).origin : ''
const GOTRUE_URL = process.env.NEXT_PUBLIC_API_URL
? new URL(process.env.NEXT_PUBLIC_GOTRUE_URL).origin
: ''
const SUPABASE_PROJECTS_URL = 'https://*.supabase.co'
const SUPABASE_PROJECTS_URL_WS = 'wss://*.supabase.co'
const SUPABASE_STAGING_PROJECTS_URL = 'https://*.supabase.red'
+3 -1
View File
@@ -147,6 +147,8 @@ function CustomApp({ Component, pageProps }: AppPropsWithLayout) {
useThemeSandbox()
const isTestEnv = process.env.NEXT_PUBLIC_NODE_ENV === 'test'
return (
<QueryClientProvider client={queryClient}>
<Hydrate state={pageProps.dehydratedState}>
@@ -181,7 +183,7 @@ function CustomApp({ Component, pageProps }: AppPropsWithLayout) {
<HCaptchaLoadedStore />
<PortalToast />
<ReactQueryDevtools initialIsOpen={false} position="bottom-right" />
{!isTestEnv && <ReactQueryDevtools initialIsOpen={false} position="bottom-right" />}
</FlagProvider>
</ProfileProvider>
</AuthContainer>
+78 -2
View File
@@ -12,6 +12,7 @@
"apps/*",
"apps/docs/spec/parser",
"tests",
"playwright-tests",
"packages/*"
],
"devDependencies": {
@@ -6306,6 +6307,21 @@
"node": ">=14"
}
},
"node_modules/@playwright/test": {
"version": "1.41.2",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.41.2.tgz",
"integrity": "sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg==",
"dev": true,
"dependencies": {
"playwright": "1.41.2"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=16"
}
},
"node_modules/@pmmmwh/react-refresh-webpack-plugin": {
"version": "0.5.11",
"dev": true,
@@ -12023,8 +12039,12 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "20.8.2",
"license": "MIT"
"version": "20.11.16",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.16.tgz",
"integrity": "sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ==",
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/@types/node-fetch": {
"version": "2.6.6",
@@ -28686,6 +28706,54 @@
"node": ">=10"
}
},
"node_modules/playwright": {
"version": "1.41.2",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.41.2.tgz",
"integrity": "sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A==",
"dev": true,
"dependencies": {
"playwright-core": "1.41.2"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=16"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.41.2",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.41.2.tgz",
"integrity": "sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA==",
"dev": true,
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=16"
}
},
"node_modules/playwright-tests": {
"resolved": "playwright-tests",
"link": true
},
"node_modules/playwright/node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/pluralize": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",
@@ -37914,6 +37982,14 @@
"dev": true,
"license": "ISC"
},
"playwright-tests": {
"version": "1.0.0",
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.41.2",
"@types/node": "^20.11.16"
}
},
"tests": {
"name": "supabase-tests",
"version": "1.0.0",
+3 -14
View File
@@ -5,12 +5,7 @@
"author": "Supabase, Inc.",
"license": "Apache-2.0",
"private": true,
"workspaces": [
"apps/*",
"apps/docs/spec/parser",
"tests",
"packages/*"
],
"workspaces": ["apps/*", "apps/docs/spec/parser", "tests", "playwright-tests", "packages/*"],
"scripts": {
"build": "turbo run build",
"build:studio": "turbo run build --filter=studio",
@@ -30,6 +25,7 @@
"test:docs": "turbo run test --filter=docs",
"test:ui": "turbo run test --filter=ui",
"test:studio": "turbo run test --filter=studio",
"test:playwright": "npm --prefix playwright-tests run test",
"perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/",
"perf:meta": "ab -t 5 -c 20 -T application/json http://localhost:5555/tables",
"generate:types": "supabase gen types typescript --local > ./supabase/functions/common/database-types.ts"
@@ -51,14 +47,7 @@
"npm": ">=9.0.0",
"node": ">=18.0.0"
},
"keywords": [
"postgres",
"firebase",
"storage",
"functions",
"database",
"auth"
],
"keywords": ["postgres", "firebase", "storage", "functions", "database", "auth"],
"volta": {
"node": "18.17.1"
}
+6
View File
@@ -0,0 +1,6 @@
node_modules/
/test-results/
/playwright-report/
/blob-report/
/playwright/.cache/
keys.json
+48
View File
@@ -0,0 +1,48 @@
# Local studio tests
In an effort to make the local studio more stable, we've added tests which test features which are commonly used in local
development. Built with [Playwright](https://playwright.dev/docs/intro).
## How to run tests
Before running the tests, make sure you've done the following:
- Run `npm install` in the root folder of this repo
- `docker` or `orbstack` is currently running
- `supabase` CLI is already [installed](https://github.com/supabase/cli?tab=readme-ov-file#install-the-cli)
- no other supabase local environment is running (infrastructure environment is ok)
You can run the tests by running `npm run test` in this folder.
When you run the command, it includes:
1. Setting up the local environment using the `supabase` CLI
2. Extracting the environment variables and saving them into a `.env.test` file in the studio app
3. Running the `studio` app in dev mode (`npm run dev`)
4. Running the tests
5. Stopping the `studio` app and the local environment
If the environment does't stop for some reason, you'll see `supabase start is already running` on the next run. In this
case, just run `supabase stop` between test runs.
## How to write tests
Playwright has a nice [Codegen tool](https://playwright.dev/docs/codegen-intro#running-codegen) which you can use to
record your actions:
```bash
npm run codegen:setup
# in a separate terminal
npm run codegen
```
## How to debug/fix tests
If you've run the tests locally and you want to see the results, Playwright has a [UI mode](https://playwright.dev/docs/test-ui-mode)
which you can use to run and replay specific tests:
```bash
npm run test -- --ui
```
It will also record any failing tests when running `npm run test` in this folder.
+21
View File
@@ -0,0 +1,21 @@
{
"name": "playwright-tests",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"env:setup": "supabase start -x studio && supabase status --output json > keys.json && node scripts/generate-env.js",
"pretest": "npm run env:setup",
"test": "playwright test",
"posttest": "supabase stop --no-backup",
"codegen:setup": "npm run env:setup && NODE_ENV=test npm --prefix ../apps/studio run dev",
"codegen": "playwright codegen http://localhost:8082/project/default"
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.41.2",
"@types/node": "^20.11.16"
}
}
+79
View File
@@ -0,0 +1,79 @@
import { defineConfig, devices } from '@playwright/test'
/**
* Read environment variables from file.
* https://github.com/motdotla/dotenv
*/
// require('dotenv').config();
/**
* See https://playwright.dev/docs/test-configuration.
*/
export default defineConfig({
testDir: './tests',
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: process.env.CI ? 2 : 0,
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html',
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
baseURL: 'http://localhost:8082',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry',
// record a video for failed tests, but only for local testing. We can't store videos on CI atm.
video: process.env.CI ? 'off' : 'retain-on-failure',
},
/* Configure projects for major browsers */
projects: [
{
name: 'chromium',
use: { ...devices['Desktop Chrome'] },
},
// {
// name: 'firefox',
// use: { ...devices['Desktop Firefox'] },
// },
// {
// name: 'webkit',
// use: { ...devices['Desktop Safari'] },
// },
/* Test against mobile viewports. */
// {
// name: 'Mobile Chrome',
// use: { ...devices['Pixel 5'] },
// },
// {
// name: 'Mobile Safari',
// use: { ...devices['iPhone 12'] },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
// },
// {
// name: 'Google Chrome',
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
// },
],
/* Run your local dev server before starting the tests */
webServer: {
command: 'NODE_ENV=test npm --prefix ../apps/studio run dev',
url: 'http://localhost:8082',
reuseExistingServer: !process.env.CI,
},
})
+37
View File
@@ -0,0 +1,37 @@
const fs = require('fs')
const generatedEnv = require('../keys.json')
/**
* This script takes the API keys from the local environment, merges them with some predefined variables and saves them
* to a env.test file in the studio app. This is needed to prepare the studio so that it can be run with the local
* environment as the backend.
*/
const defaultEnv = {
// POSTGRES_PASSWORD: 'postgres',
// NEXT_ANALYTICS_BACKEND_PROVIDER: 'postgres',
// SUPABASE_REST_URL: 'http://127.0.0.1:54321/rest/v1/',
// NEXT_PUBLIC_ENABLE_LOGS: 'false',
// NEXT_PUBLIC_IS_PLATFORM: 'false',
SUPABASE_ANON_KEY: '$ANON_KEY',
SUPABASE_SERVICE_KEY: '$SERVICE_ROLE_KEY',
SUPABASE_URL: '$API_URL',
STUDIO_PG_META_URL: '$API_URL/pg',
SUPABASE_PUBLIC_URL: '$API_URL',
SENTRY_IGNORE_API_RESOLUTION_ERROR: '1',
LOGFLARE_URL: 'http://localhost:54329',
LOGFLARE_API_KEY: 'api-key',
NEXT_PUBLIC_SITE_URL: 'http://localhost:3000',
NEXT_PUBLIC_GOTRUE_URL: '$SUPABASE_PUBLIC_URL/auth/v1',
NEXT_PUBLIC_HCAPTCHA_SITE_KEY: '10000000-ffff-ffff-ffff-000000000001',
NEXT_PUBLIC_NODE_ENV: 'test',
}
const environment = { ...generatedEnv, ...defaultEnv }
fs.writeFileSync(
'../apps/studio/.env.test',
Object.keys(environment)
.map((key) => `${key}=${environment[key]}`)
.join('\n')
)
+4
View File
@@ -0,0 +1,4 @@
# Supabase
.branches
.temp
.env
+159
View File
@@ -0,0 +1,159 @@
# A string used to distinguish different Supabase projects on the same host. Defaults to the
# working directory name when running `supabase init`.
project_id = "playwright-tests"
[api]
enabled = true
# Port to use for the API URL.
port = 54321
# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
# endpoints. public and storage are always included.
schemas = ["public", "storage", "graphql_public"]
# Extra schemas to add to the search_path of every request. public is always included.
extra_search_path = ["public", "extensions"]
# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
# for accidental or malicious requests.
max_rows = 1000
[db]
# Port to use for the local database URL.
port = 54322
# Port used by db diff command to initialize the shadow database.
shadow_port = 54320
# The database major version to use. This has to be the same as your remote database's. Run `SHOW
# server_version;` on the remote database to check.
major_version = 15
[db.pooler]
enabled = false
# Port to use for the local connection pooler.
port = 54329
# Specifies when a server connection can be reused by other clients.
# Configure one of the supported pooler modes: `transaction`, `session`.
pool_mode = "transaction"
# How many server connections to allow per user/database pair.
default_pool_size = 20
# Maximum number of client connections allowed.
max_client_conn = 100
[realtime]
enabled = true
# Bind realtime via either IPv4 or IPv6. (default: IPv6)
# ip_version = "IPv6"
# The maximum length in bytes of HTTP request headers. (default: 4096)
# max_header_length = 4096
[studio]
enabled = true
# Port to use for Supabase Studio.
port = 54323
# External URL of the API server that frontend connects to.
api_url = "http://127.0.0.1"
# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
# are monitored, and you can view the emails that would have been sent from the web interface.
[inbucket]
enabled = true
# Port to use for the email testing server web interface.
port = 54324
# Uncomment to expose additional ports for testing user applications that send emails.
# smtp_port = 54325
# pop3_port = 54326
[storage]
enabled = true
# The maximum file size allowed (e.g. "5MB", "500KB").
file_size_limit = "50MiB"
[auth]
enabled = true
# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
# in emails.
site_url = "http://127.0.0.1:3000"
# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
additional_redirect_urls = ["https://127.0.0.1:3000"]
# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week).
jwt_expiry = 3600
# If disabled, the refresh token will never expire.
enable_refresh_token_rotation = true
# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds.
# Requires enable_refresh_token_rotation = true.
refresh_token_reuse_interval = 10
# Allow/disallow new user signups to your project.
enable_signup = true
# Allow/disallow testing manual linking of accounts
# enable_manual_linking = false
[auth.email]
# Allow/disallow new user signups via email to your project.
enable_signup = true
# If enabled, a user will be required to confirm any email change on both the old, and new email
# addresses. If disabled, only the new email is required to confirm.
double_confirm_changes = true
# If enabled, users need to confirm their email address before signing in.
enable_confirmations = false
# Uncomment to customize email template
# [auth.email.template.invite]
# subject = "You have been invited"
# content_path = "./supabase/templates/invite.html"
[auth.sms]
# Allow/disallow new user signups via SMS to your project.
enable_signup = true
# If enabled, users need to confirm their phone number before signing in.
enable_confirmations = false
# Template for sending OTP to users
# template = "Your code is {{ .Code }} ."
# Use pre-defined map of phone number to OTP for testing.
[auth.sms.test_otp]
# 4152127777 = "123456"
# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used.
# [auth.hook.custom_access_token]
# enabled = true
# uri = "pg-functions://<database>/<schema>/<hook_name>"
# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`.
[auth.sms.twilio]
enabled = false
account_sid = ""
message_service_sid = ""
# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead:
auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)"
# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`,
# `twitter`, `slack`, `spotify`, `workos`, `zoom`.
[auth.external.apple]
enabled = false
client_id = ""
# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead:
secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)"
# Overrides the default auth redirectUrl.
redirect_uri = ""
# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure,
# or any other third-party OIDC providers.
url = ""
[analytics]
enabled = false
port = 54327
vector_port = 54328
# Configure one of the supported backends: `postgres`, `bigquery`.
backend = "postgres"
# Experimental features may be deprecated any time
# [experimental]
# Configures Postgres storage engine to use OrioleDB (S3)
# orioledb_version = ""
# Configures S3 bucket URL, eg. <bucket_name>.s3-<region>.amazonaws.com
# s3_host = "env(S3_HOST)"
# Configures S3 bucket region, eg. us-east-1
# s3_region = "env(S3_REGION)"
# Configures AWS_ACCESS_KEY_ID for S3 bucket
# s3_access_key = "env(S3_ACCESS_KEY)"
# Configures AWS_SECRET_ACCESS_KEY for S3 bucket
# s3_secret_key = "env(S3_SECRET_KEY)"
View File
@@ -0,0 +1,30 @@
import { expect, test } from '@playwright/test'
test.describe('Table Editor page', () => {
test('should create a column and insert a row', async ({ page }) => {
const name = 'TestTable' + Math.floor(Math.random() * 10)
await page.goto('/project/default/editor')
await page.getByRole('button', { name: 'New table', exact: true }).click()
// need to wait for the panel animation
await page.waitForTimeout(1000)
await page.locator('.col-span-8 > div > .relative > .peer\\/input').first().click()
await page.locator('.col-span-8 > div > .relative > .peer\\/input').first().fill(name)
await page.getByRole('button', { name: 'Add column' }).click()
await page.getByRole('textbox', { name: 'column_name' }).click()
await page.getByRole('textbox', { name: 'column_name' }).fill('textColumn')
await page.getByRole('button', { name: '---' }).click()
await page.getByText('textVariable-length character').click()
await page.waitForTimeout(1000)
await page.getByRole('button', { name: 'Save' }).click()
await page.locator('#toast').getByRole('button').click()
await page.getByRole('link', { name }).click()
await page.getByTestId('table-editor-insert-new-row').click()
await page.getByText('Insert a new row into').click()
await page.getByPlaceholder('NULL').click()
await page.getByPlaceholder('NULL').fill('some text')
await page.getByTestId('action-bar-save-row').click()
await page.locator('#toast').getByRole('button').click()
await expect(page.getByRole('grid')).toContainText('some text')
})
})