diff --git a/templates/keynote-2/.env.example b/templates/keynote-2/.env.example index faa3e1fa7..cf8a45c2c 100644 --- a/templates/keynote-2/.env.example +++ b/templates/keynote-2/.env.example @@ -1,10 +1,10 @@ # ===== Runtime toggles ===== -USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip +#USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip #SKIP_PG=1 # 1 = don't init Postgres in prep #SKIP_CRDB=1 # 1 = don't init Cockroach in prep #SKIP_SQLITE=1 # 1 = don't init SQLite in prep #SKIP_SUPABASE=1 # 1 = don't init Supabase in prep -SKIP_CONVEX=1 # 1 = don't init Convex in prep +#SKIP_CONVEX=1 # 1 = don't init Convex in prep SPACETIME_METRICS_ENDPOINT=0 # ===== PostgreSQL ===== @@ -40,7 +40,7 @@ SUPABASE_DB_URL=postgresql://postgres:postgres@127.0.0.1:54322/postgres CONVEX_URL=http://127.0.0.1:3210 CONVEX_SITE_URL=http://127.0.0.1:3210 CLEAR_CONVEX_ON_PREP=0 -CONVEX_USE_SHARDED_COUNTER=1 +#CONVEX_USE_SHARDED_COUNTER=1 # ===== Bun ===== BUN_URL=http://127.0.0.1:4001 diff --git a/templates/keynote-2/DEVELOP.md b/templates/keynote-2/DEVELOP.md index fc4deee14..1b0d0d94d 100644 --- a/templates/keynote-2/DEVELOP.md +++ b/templates/keynote-2/DEVELOP.md @@ -15,7 +15,7 @@ Each run: Run a quick performance comparison: ```bash -npm run demo +pnpm run demo ``` The script will: @@ -44,6 +44,7 @@ The script will: - **Node.js** ≥ 22.x - **pnpm** installed globally +- **Rust** (required for SpacetimeDB benchmarks) -- [install](https://rust-lang.org/tools/install/) - **Docker** for local Postgres / Cockroach / Supabase - Local/Cloud Convex @@ -160,42 +161,42 @@ cd .. 1. Start SpacetimeDB (`cargo run -p spacetimedb-cli -- start` or `spacetime start`) 2. Start Convex (inside convex-app run `npx convex dev`) 3. Init Supabase (run `supabase init`) inside project root. -4. `npm run prep` to seed the databases. -5. `npm run bench` to run the test against all connectors. +4. `pnpm run prep` to seed the databases. +5. `pnpm run bench` to run the test against all connectors. ## Commands & Examples ### 1. Run a test ```bash -npm run bench -- [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] [--stdb-compression none|gzip] +pnpm run bench [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] [--stdb-compression none|gzip] ``` Examples: ```bash # Default test (test-1), default args -npm run bench +pnpm run bench # Explicit test name -npm run bench -- test-1 +pnpm run bench test-1 # Short run, 100 concurrent workers -npm run bench -- test-1 --seconds 10 --concurrency 100 +pnpm run bench test-1 --seconds 10 --concurrency 100 # Heavier skew on hot accounts -npm run bench -- test-1 --alpha 2.0 +pnpm run bench test-1 --alpha 2.0 # Enable gzip for the SpacetimeDB benchmark client -npm run bench -- test-1 --connectors spacetimedb --stdb-compression gzip +pnpm run bench test-1 --connectors spacetimedb --stdb-compression gzip # Only run selected connectors -npm run bench -- test-1 --connectors spacetimedb,sqlite_rpc +pnpm run bench test-1 --connectors spacetimedb,sqlite_rpc ``` ### 2. Run the distributed TypeScript SpacetimeDB benchmark -Use this mode when you want to spread explicit TypeScript client connections across multiple machines. The existing `npm run bench` flow is still the single-process benchmark; the distributed flow is a separate coordinator + generator setup. +Use this mode when you want to spread explicit TypeScript client connections across multiple machines. The existing `pnpm run bench` flow is still the single-process benchmark; the distributed flow is a separate coordinator + generator setup. The commands below are written so they run unchanged on a single machine. For a true multi-machine run, replace `127.0.0.1` with the actual coordinator and server hostnames or IP addresses reachable from each generator machine. @@ -387,11 +388,11 @@ From `src/cli.ts`: - **`--seconds N`** - Duration of the benchmark in seconds - - Default: `1` + - Default: `10` - **`--concurrency N`** - Number of workers / in-flight operations - - Default: `10` + - Default: `50` - **`--alpha A`** - Zipf α parameter for account selection (hot vs cold distribution) @@ -407,6 +408,7 @@ From `src/cli.ts`: - If omitted, all connectors for that test are run - The valid names come from `tc.system` in the test modules and the keys in `CONNECTORS` + - Valid names: `convex`, `spacetimedb`, `bun`, `postgres_rpc`, `cockroach_rpc`, `sqlite_rpc`, `supabase_rpc`, `planetscale_pg_rpc` - **`--contention-tests startAlpha endAlpha step concurrency`** - Runs a sweep over Zipf α values for a single connector @@ -425,14 +427,10 @@ From `src/cli.ts`: You can also run the benchmark via Docker instead of Node directly: ```bash -docker compose run --rm bench \ - --seconds 5 \ - --concurrency 50 \ - --alpha 1 \ - --connectors convex +docker compose run --rm bench -- --seconds 5 --concurrency 50 --alpha 1 --connectors convex ``` -If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `npm run prep` before running bench. +If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `pnpm run prep` before running bench. ## Output diff --git a/templates/keynote-2/LICENSE b/templates/keynote-2/LICENSE new file mode 120000 index 000000000..039e117dd --- /dev/null +++ b/templates/keynote-2/LICENSE @@ -0,0 +1 @@ +../../licenses/apache2.txt \ No newline at end of file diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 967f7f8ad..1d452f993 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -8,12 +8,12 @@ See SpacetimeDB's performance advantage with one command: ```bash pnpm install -npm run demo +pnpm run demo ``` The demo compares SpacetimeDB and Convex by default, since both are easy for anyone to set up and run locally without additional infrastructure. Other systems (Postgres, CockroachDB, SQLite, etc.) are also supported but require more setup. The demo checks that required services are running (prompts you to start them if not), seeds databases, and displays animated results. -**Options:** `--systems a,b,c` | `--seconds N` | `--skip-prep` | `--no-animation` +**Options:** `--systems a,b,c` | `--seconds N` | `--concurrency N` | `--alpha N` | `--skip-prep` | `--no-animation` **Note:** `demo` always runs the built-in `test-1` scenario. Use `bench` if you need to specify a test name directly. **Note:** `demo` selects targets with `--systems`; `bench` filters test connectors with `--connectors`. @@ -84,7 +84,7 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command ```bash -docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connectors YY +docker compose run --rm bench -- --seconds 10 --concurrency 50 --alpha XX --connectors YY ``` - `--seconds 10`: Duration of benchmark run @@ -97,7 +97,7 @@ docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connect **Server Machine (Variant A - PhoenixNAP):** -- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory OS: Ubuntu 24.04 +- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 **Server Machine (Variant B - Google Cloud):** @@ -174,28 +174,9 @@ PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud dat ## Running the Benchmarks -See [DEVELOP.md](./DEVELOP.md) for detailed setup and execution instructions. +See [DEVELOP.md](./DEVELOP.md) for prerequisites, configuration, and full CLI reference. The distributed TypeScript SpacetimeDB workflow is documented there as `Run the distributed TypeScript SpacetimeDB benchmark`. -### Quick Start - -```bash -# Install dependencies -pnpm install - -# Copy environment config -cp .env.example .env - -# Start services (PostgreSQL, CockroachDB, etc.) -docker compose up -d pg crdb - -# Seed databases -npm run prep - -# Run benchmark -npm run bench -- test-1 --seconds 10 --concurrency 50 --alpha 1.5 --connectors spacetimedb,postgres_rpc,sqlite_rpc -``` - ## Output Benchmark results are written to `./runs/` as JSON files with TPS and latency statistics. diff --git a/templates/keynote-2/src/opts.ts b/templates/keynote-2/src/opts.ts index fd265d170..bd17ebd60 100644 --- a/templates/keynote-2/src/opts.ts +++ b/templates/keynote-2/src/opts.ts @@ -306,7 +306,7 @@ export function parseDemoOptions(argv: string[] = process.argv): DemoOptions { return { ...runtimeOptions, seconds: options.seconds ?? 10, - concurrency: options.concurrency ?? 10, + concurrency: options.concurrency ?? 50, alpha: options.alpha ?? 1.5, systems: options.systems ?? options.connectors ?? [...defaultDemoSystems], @@ -378,7 +378,7 @@ export function parseBenchOptions(argv: string[] = process.argv): BenchOptions { return { ...runtimeOptions, testName: args[0] ?? defaultBenchTestName, - seconds: options.seconds ?? 1, + seconds: options.seconds ?? 10, concurrency: contentionTests?.concurrency ?? options.concurrency ?? 50, alpha: concurrencyTests?.alpha ?? options.alpha ?? 1.5,