From 7143ed9722d18112e2546a47dc3b97516f2d1128 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Fri, 17 Apr 2026 09:03:36 -0400 Subject: [PATCH] Clean up keynote-2 template README & DEVELOP (#4624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description of Changes Refresh this stale PR against current `master`. Several original items were already applied upstream or conflict with recent keynote-2 work (#4616, #4647, #4678, #4682, #4698, #4703, #4743, #4745, #4753, #4757), so those are dropped. What remains is the still-relevant subset, rebased onto the current file structures. **README.md:** - Use `pnpm run demo` in Quick Demo (consistency with pnpm workspace) - Add `--concurrency` and `--alpha` to demo options - Add `--` separator to `docker compose run` bench example - Fix hardware config punctuation (add comma before "OS:") - Remove redundant Quick Start section; replace with link to DEVELOP.md for prerequisites and CLI reference - Add symlink for license **DEVELOP.md:** - Use `pnpm run` throughout (demo, prep, bench) instead of `npm run` - Drop the `-- ` pass-through after `pnpm run bench` (not needed with pnpm; matches the `#4703` testing examples) - Add Rust to Prerequisites - Add explicit list of valid connector names (`convex`, `spacetimedb`, `bun`, `postgres_rpc`, `cockroach_rpc`, `sqlite_rpc`, `supabase_rpc`, `planetscale_pg_rpc`) - Update CLI reference defaults to match methodology (seconds: 1→10, concurrency: 10→50) - Condense `docker compose run` bench example to a single line with `--` separator; fix `npm prep` → `pnpm run prep` **src/opts.ts:** (CLI parsing moved here in `#4703`; original PR targeted the now-gone inline parsers in `cli.ts`/`demo.ts`) - `parseBenchOptions`: bench `--seconds` default `1` → `10` - `parseDemoOptions`: demo `--concurrency` default `10` → `50` **.env.example:** - Comment out `USE_DOCKER=1` and `SKIP_CONVEX=1` so demo defaults (convex, spacetimedb) work out of the box - Comment out `CONVEX_USE_SHARDED_COUNTER=1` (still a supported knob, just off by default) # Dropped as superseded by master - Rust Client README section tweaks (heading capitalization, `bottlnecked`/`then` typo fixes) — section was removed by `#4753` - Rename `SPACETIME_METRICS_ENDPOINT` → `USE_SPACETIME_METRICS_ENDPOINT` — master's `src/config.ts` still reads the original name - Connector-name fixes in examples (`sqlite` → `sqlite_rpc`, `postgres` → `postgres_rpc`) — already corrected on master # API and ABI breaking changes None. # Expected complexity level and risk **1** – Documentation and default-value changes. No functional changes to core logic. # Testing - [x] `pnpm install` in `templates/keynote-2/` succeeds - [x] `pnpm run bench --help` / `pnpm run demo --help` render with valid-connec --- templates/keynote-2/.env.example | 6 +++--- templates/keynote-2/DEVELOP.md | 36 +++++++++++++++----------------- templates/keynote-2/LICENSE | 1 + templates/keynote-2/README.md | 29 +++++-------------------- templates/keynote-2/src/opts.ts | 4 ++-- 5 files changed, 28 insertions(+), 48 deletions(-) create mode 120000 templates/keynote-2/LICENSE diff --git a/templates/keynote-2/.env.example b/templates/keynote-2/.env.example index faa3e1fa7..cf8a45c2c 100644 --- a/templates/keynote-2/.env.example +++ b/templates/keynote-2/.env.example @@ -1,10 +1,10 @@ # ===== Runtime toggles ===== -USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip +#USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip #SKIP_PG=1 # 1 = don't init Postgres in prep #SKIP_CRDB=1 # 1 = don't init Cockroach in prep #SKIP_SQLITE=1 # 1 = don't init SQLite in prep #SKIP_SUPABASE=1 # 1 = don't init Supabase in prep -SKIP_CONVEX=1 # 1 = don't init Convex in prep +#SKIP_CONVEX=1 # 1 = don't init Convex in prep SPACETIME_METRICS_ENDPOINT=0 # ===== PostgreSQL ===== @@ -40,7 +40,7 @@ SUPABASE_DB_URL=postgresql://postgres:postgres@127.0.0.1:54322/postgres CONVEX_URL=http://127.0.0.1:3210 CONVEX_SITE_URL=http://127.0.0.1:3210 CLEAR_CONVEX_ON_PREP=0 -CONVEX_USE_SHARDED_COUNTER=1 +#CONVEX_USE_SHARDED_COUNTER=1 # ===== Bun ===== BUN_URL=http://127.0.0.1:4001 diff --git a/templates/keynote-2/DEVELOP.md b/templates/keynote-2/DEVELOP.md index fc4deee14..1b0d0d94d 100644 --- a/templates/keynote-2/DEVELOP.md +++ b/templates/keynote-2/DEVELOP.md @@ -15,7 +15,7 @@ Each run: Run a quick performance comparison: ```bash -npm run demo +pnpm run demo ``` The script will: @@ -44,6 +44,7 @@ The script will: - **Node.js** ≥ 22.x - **pnpm** installed globally +- **Rust** (required for SpacetimeDB benchmarks) -- [install](https://rust-lang.org/tools/install/) - **Docker** for local Postgres / Cockroach / Supabase - Local/Cloud Convex @@ -160,42 +161,42 @@ cd .. 1. Start SpacetimeDB (`cargo run -p spacetimedb-cli -- start` or `spacetime start`) 2. Start Convex (inside convex-app run `npx convex dev`) 3. Init Supabase (run `supabase init`) inside project root. -4. `npm run prep` to seed the databases. -5. `npm run bench` to run the test against all connectors. +4. `pnpm run prep` to seed the databases. +5. `pnpm run bench` to run the test against all connectors. ## Commands & Examples ### 1. Run a test ```bash -npm run bench -- [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] [--stdb-compression none|gzip] +pnpm run bench [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] [--stdb-compression none|gzip] ``` Examples: ```bash # Default test (test-1), default args -npm run bench +pnpm run bench # Explicit test name -npm run bench -- test-1 +pnpm run bench test-1 # Short run, 100 concurrent workers -npm run bench -- test-1 --seconds 10 --concurrency 100 +pnpm run bench test-1 --seconds 10 --concurrency 100 # Heavier skew on hot accounts -npm run bench -- test-1 --alpha 2.0 +pnpm run bench test-1 --alpha 2.0 # Enable gzip for the SpacetimeDB benchmark client -npm run bench -- test-1 --connectors spacetimedb --stdb-compression gzip +pnpm run bench test-1 --connectors spacetimedb --stdb-compression gzip # Only run selected connectors -npm run bench -- test-1 --connectors spacetimedb,sqlite_rpc +pnpm run bench test-1 --connectors spacetimedb,sqlite_rpc ``` ### 2. Run the distributed TypeScript SpacetimeDB benchmark -Use this mode when you want to spread explicit TypeScript client connections across multiple machines. The existing `npm run bench` flow is still the single-process benchmark; the distributed flow is a separate coordinator + generator setup. +Use this mode when you want to spread explicit TypeScript client connections across multiple machines. The existing `pnpm run bench` flow is still the single-process benchmark; the distributed flow is a separate coordinator + generator setup. The commands below are written so they run unchanged on a single machine. For a true multi-machine run, replace `127.0.0.1` with the actual coordinator and server hostnames or IP addresses reachable from each generator machine. @@ -387,11 +388,11 @@ From `src/cli.ts`: - **`--seconds N`** - Duration of the benchmark in seconds - - Default: `1` + - Default: `10` - **`--concurrency N`** - Number of workers / in-flight operations - - Default: `10` + - Default: `50` - **`--alpha A`** - Zipf α parameter for account selection (hot vs cold distribution) @@ -407,6 +408,7 @@ From `src/cli.ts`: - If omitted, all connectors for that test are run - The valid names come from `tc.system` in the test modules and the keys in `CONNECTORS` + - Valid names: `convex`, `spacetimedb`, `bun`, `postgres_rpc`, `cockroach_rpc`, `sqlite_rpc`, `supabase_rpc`, `planetscale_pg_rpc` - **`--contention-tests startAlpha endAlpha step concurrency`** - Runs a sweep over Zipf α values for a single connector @@ -425,14 +427,10 @@ From `src/cli.ts`: You can also run the benchmark via Docker instead of Node directly: ```bash -docker compose run --rm bench \ - --seconds 5 \ - --concurrency 50 \ - --alpha 1 \ - --connectors convex +docker compose run --rm bench -- --seconds 5 --concurrency 50 --alpha 1 --connectors convex ``` -If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `npm run prep` before running bench. +If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `pnpm run prep` before running bench. ## Output diff --git a/templates/keynote-2/LICENSE b/templates/keynote-2/LICENSE new file mode 120000 index 000000000..039e117dd --- /dev/null +++ b/templates/keynote-2/LICENSE @@ -0,0 +1 @@ +../../licenses/apache2.txt \ No newline at end of file diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 967f7f8ad..1d452f993 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -8,12 +8,12 @@ See SpacetimeDB's performance advantage with one command: ```bash pnpm install -npm run demo +pnpm run demo ``` The demo compares SpacetimeDB and Convex by default, since both are easy for anyone to set up and run locally without additional infrastructure. Other systems (Postgres, CockroachDB, SQLite, etc.) are also supported but require more setup. The demo checks that required services are running (prompts you to start them if not), seeds databases, and displays animated results. -**Options:** `--systems a,b,c` | `--seconds N` | `--skip-prep` | `--no-animation` +**Options:** `--systems a,b,c` | `--seconds N` | `--concurrency N` | `--alpha N` | `--skip-prep` | `--no-animation` **Note:** `demo` always runs the built-in `test-1` scenario. Use `bench` if you need to specify a test name directly. **Note:** `demo` selects targets with `--systems`; `bench` filters test connectors with `--connectors`. @@ -84,7 +84,7 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command ```bash -docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connectors YY +docker compose run --rm bench -- --seconds 10 --concurrency 50 --alpha XX --connectors YY ``` - `--seconds 10`: Duration of benchmark run @@ -97,7 +97,7 @@ docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connect **Server Machine (Variant A - PhoenixNAP):** -- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory OS: Ubuntu 24.04 +- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 **Server Machine (Variant B - Google Cloud):** @@ -174,28 +174,9 @@ PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud dat ## Running the Benchmarks -See [DEVELOP.md](./DEVELOP.md) for detailed setup and execution instructions. +See [DEVELOP.md](./DEVELOP.md) for prerequisites, configuration, and full CLI reference. The distributed TypeScript SpacetimeDB workflow is documented there as `Run the distributed TypeScript SpacetimeDB benchmark`. -### Quick Start - -```bash -# Install dependencies -pnpm install - -# Copy environment config -cp .env.example .env - -# Start services (PostgreSQL, CockroachDB, etc.) -docker compose up -d pg crdb - -# Seed databases -npm run prep - -# Run benchmark -npm run bench -- test-1 --seconds 10 --concurrency 50 --alpha 1.5 --connectors spacetimedb,postgres_rpc,sqlite_rpc -``` - ## Output Benchmark results are written to `./runs/` as JSON files with TPS and latency statistics. diff --git a/templates/keynote-2/src/opts.ts b/templates/keynote-2/src/opts.ts index fd265d170..bd17ebd60 100644 --- a/templates/keynote-2/src/opts.ts +++ b/templates/keynote-2/src/opts.ts @@ -306,7 +306,7 @@ export function parseDemoOptions(argv: string[] = process.argv): DemoOptions { return { ...runtimeOptions, seconds: options.seconds ?? 10, - concurrency: options.concurrency ?? 10, + concurrency: options.concurrency ?? 50, alpha: options.alpha ?? 1.5, systems: options.systems ?? options.connectors ?? [...defaultDemoSystems], @@ -378,7 +378,7 @@ export function parseBenchOptions(argv: string[] = process.argv): BenchOptions { return { ...runtimeOptions, testName: args[0] ?? defaultBenchTestName, - seconds: options.seconds ?? 1, + seconds: options.seconds ?? 10, concurrency: contentionTests?.concurrency ?? options.concurrency ?? 50, alpha: concurrencyTests?.alpha ?? options.alpha ?? 1.5,