Improve benchmark cli, make compatible with deno (#4647)

# Description of Changes

Now we get a `--help` for the benchmark, which is nicer. Also now can
run under deno, with `deno --sloppy-imports -A src/demo.ts` (might be
useful, deno's websocket is implemented in native code while node's is
implemented in JS). I removed the
[BOM](https://en.wikipedia.org/wiki/Byte_order_mark) because it seems
unintentional (only found in `templates/keynote-2`) and was causing a
little bit of weirdness.

Also, fix the rust benchmark client as a follow-up to #4616 

# Expected complexity level and risk

1

# Testing

- [x] Works under deno and has usage
This commit is contained in:
Noa
2026-03-23 21:25:24 -05:00
committed by GitHub
parent 05a4a7ba83
commit 7d0a0b97d0
55 changed files with 523 additions and 314 deletions
+64 -25
View File
@@ -362,7 +362,7 @@ importers:
devDependencies:
'@types/bun':
specifier: latest
version: 1.3.9
version: 1.3.10
bun:
specifier: ^1.3.2
version: 1.3.9
@@ -458,6 +458,9 @@ importers:
bun:
specifier: ^1.3.2
version: 1.3.9
cac:
specifier: ^7.0.0
version: 7.0.0
drizzle-orm:
specifier: ^0.44.7
version: 0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)
@@ -521,7 +524,7 @@ importers:
dependencies:
nuxt:
specifier: ~3.16.0
version: 3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2)
version: 3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2)
spacetimedb:
specifier: workspace:*
version: link:../../crates/bindings-typescript
@@ -5914,6 +5917,9 @@ packages:
'@types/bonjour@3.5.13':
resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==}
'@types/bun@1.3.10':
resolution: {integrity: sha512-0+rlrUrOrTSskibryHbvQkDOWRJwJZqZlxrUs1u4oOoTln8+WIXBPmAuCF35SWB2z4Zl3E84Nl/D0P7803nigQ==}
'@types/bun@1.3.9':
resolution: {integrity: sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw==}
@@ -7275,11 +7281,15 @@ packages:
buffer@6.0.3:
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
bun-types@1.3.10:
resolution: {integrity: sha512-tcpfCCl6XWo6nCVnpcVrxQ+9AYN1iqMIzgrSKYMB/fjLtV2eyAVEg7AxQJuCq/26R6HpKWykQXuSOq/21RYcbg==}
bun-types@1.3.9:
resolution: {integrity: sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg==}
bun@1.3.9:
resolution: {integrity: sha512-v5hkh1us7sMNjfimWE70flYbD5I1/qWQaqmJ45q2qk5H/7muQVa478LSVRSFyGTBUBog2LsPQnfIRdjyWJRY+A==}
cpu: [arm64, x64]
os: [darwin, linux, win32]
hasBin: true
@@ -7320,6 +7330,10 @@ packages:
resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==}
engines: {node: '>=8'}
cac@7.0.0:
resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==}
engines: {node: '>=20.19.0'}
cacache@17.1.4:
resolution: {integrity: sha512-/aJwG2l3ZMJ1xNAnqbMpA40of9dj/pIH3QfiuQSqjfPJF747VR0J/bHn+/KdNnHKc6XQcWt/AfRSBft82W1d2A==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -8974,11 +8988,13 @@ packages:
glob@10.4.5:
resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
hasBin: true
glob@11.0.3:
resolution: {integrity: sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==}
engines: {node: 20 || >=22}
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
hasBin: true
glob@13.0.0:
@@ -13639,6 +13655,7 @@ packages:
unplugin-vue-router@0.12.0:
resolution: {integrity: sha512-xjgheKU0MegvXQcy62GVea0LjyOdMxN0/QH+ijN29W62ZlMhG7o7K+0AYqfpprvPwpWtuRjiyC5jnV2SxWye2w==}
deprecated: 'Merged into vuejs/router. Migrate: https://router.vuejs.org/guide/migration/v4-to-v5.html'
peerDependencies:
vue-router: ^4.4.0
peerDependenciesMeta:
@@ -19394,11 +19411,11 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@nuxt/cli@3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.5.1)':
'@nuxt/cli@3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.3.5)':
dependencies:
'@bomb.sh/tab': 0.0.12(cac@6.7.14)(citty@0.2.0)
'@clack/prompts': 1.0.0
c12: 3.3.3(magicast@0.5.1)
c12: 3.3.3(magicast@0.3.5)
citty: 0.2.0
confbox: 0.2.4
consola: 3.4.2
@@ -19494,9 +19511,9 @@ snapshots:
- utf-8-validate
- vue
'@nuxt/kit@3.16.2(magicast@0.5.1)':
'@nuxt/kit@3.16.2(magicast@0.3.5)':
dependencies:
c12: 3.3.3(magicast@0.5.1)
c12: 3.3.3(magicast@0.3.5)
consola: 3.4.2
defu: 6.1.4
destr: 2.0.5
@@ -19554,18 +19571,18 @@ snapshots:
pathe: 2.0.3
std-env: 3.10.0
'@nuxt/telemetry@2.7.0(@nuxt/kit@3.16.2(magicast@0.5.1))':
'@nuxt/telemetry@2.7.0(@nuxt/kit@3.16.2(magicast@0.3.5))':
dependencies:
'@nuxt/kit': 3.16.2(magicast@0.5.1)
'@nuxt/kit': 3.16.2(magicast@0.3.5)
citty: 0.2.0
consola: 3.4.2
ofetch: 2.0.0-alpha.3
rc9: 3.0.0
std-env: 3.10.0
'@nuxt/vite-builder@3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)':
'@nuxt/vite-builder@3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)':
dependencies:
'@nuxt/kit': 3.16.2(magicast@0.5.1)
'@nuxt/kit': 3.16.2(magicast@0.3.5)
'@rollup/plugin-replace': 6.0.3(rollup@4.56.0)
'@vitejs/plugin-vue': 5.2.4(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3))
'@vitejs/plugin-vue-jsx': 4.2.0(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3))
@@ -21607,6 +21624,10 @@ snapshots:
dependencies:
'@types/node': 22.18.0
'@types/bun@1.3.10':
dependencies:
bun-types: 1.3.10
'@types/bun@1.3.9':
dependencies:
bun-types: 1.3.9
@@ -23925,6 +23946,10 @@ snapshots:
base64-js: 1.5.1
ieee754: 1.2.1
bun-types@1.3.10:
dependencies:
'@types/node': 22.18.0
bun-types@1.3.9:
dependencies:
'@types/node': 22.18.0
@@ -23999,6 +24024,8 @@ snapshots:
cac@6.7.14: {}
cac@7.0.0: {}
cacache@17.1.4:
dependencies:
'@npmcli/fs': 3.1.1
@@ -24668,10 +24695,10 @@ snapshots:
whatwg-mimetype: 4.0.0
whatwg-url: 14.2.0
db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)):
db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)):
optionalDependencies:
better-sqlite3: 12.6.2
drizzle-orm: 0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)
drizzle-orm: 0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)
de-indent@1.0.2: {}
@@ -24844,6 +24871,18 @@ snapshots:
dotenv@17.2.3: {}
drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0):
optionalDependencies:
'@opentelemetry/api': 1.9.0
'@types/better-sqlite3': 7.6.13
'@types/pg': 8.16.0
'@types/sql.js': 1.4.9
better-sqlite3: 12.6.2
bun-types: 1.3.10
pg: 8.18.0
sql.js: 1.14.0
optional: true
drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0):
optionalDependencies:
'@opentelemetry/api': 1.9.0
@@ -28089,7 +28128,7 @@ snapshots:
neo-async@2.6.2: {}
nitropack@2.13.1(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13):
nitropack@2.13.1(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13):
dependencies:
'@cloudflare/kv-asset-handler': 0.4.2
'@rollup/plugin-alias': 6.0.0(rollup@4.56.0)
@@ -28110,7 +28149,7 @@ snapshots:
cookie-es: 2.0.0
croner: 9.1.0
crossws: 0.3.5
db0: 0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))
db0: 0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))
defu: 6.1.4
destr: 2.0.5
dot-prop: 10.1.0
@@ -28156,7 +28195,7 @@ snapshots:
unenv: 2.0.0-rc.24
unimport: 5.6.0
unplugin-utils: 0.3.1
unstorage: 1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2)
unstorage: 1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2)
untyped: 2.0.0
unwasm: 0.5.3
youch: 4.1.0-beta.13
@@ -28364,19 +28403,19 @@ snapshots:
schema-utils: 3.3.0
webpack: 5.102.0
nuxt@3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2):
nuxt@3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2):
dependencies:
'@nuxt/cli': 3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.5.1)
'@nuxt/cli': 3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.3.5)
'@nuxt/devalue': 2.0.2
'@nuxt/devtools': 2.7.0(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3))
'@nuxt/kit': 3.16.2(magicast@0.5.1)
'@nuxt/kit': 3.16.2(magicast@0.3.5)
'@nuxt/schema': 3.16.2
'@nuxt/telemetry': 2.7.0(@nuxt/kit@3.16.2(magicast@0.5.1))
'@nuxt/vite-builder': 3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)
'@nuxt/telemetry': 2.7.0(@nuxt/kit@3.16.2(magicast@0.3.5))
'@nuxt/vite-builder': 3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)
'@oxc-parser/wasm': 0.60.0
'@unhead/vue': 2.1.4(vue@3.5.26(typescript@5.6.3))
'@vue/shared': 3.5.26
c12: 3.3.3(magicast@0.5.1)
c12: 3.3.3(magicast@0.3.5)
chokidar: 4.0.3
compatx: 0.1.8
consola: 3.4.2
@@ -28401,7 +28440,7 @@ snapshots:
mlly: 1.8.0
mocked-exports: 0.1.1
nanotar: 0.2.1
nitropack: 2.13.1(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)
nitropack: 2.13.1(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)
nypm: 0.6.4
ofetch: 1.5.1
ohash: 2.0.11
@@ -28423,7 +28462,7 @@ snapshots:
unimport: 4.2.0
unplugin: 2.3.11
unplugin-vue-router: 0.12.0(vue-router@4.6.4(vue@3.5.26(typescript@5.6.3)))(vue@3.5.26(typescript@5.6.3))
unstorage: 1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2)
unstorage: 1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2)
untyped: 2.0.0
vue: 3.5.26(typescript@5.6.3)
vue-bundle-renderer: 2.2.0
@@ -31671,7 +31710,7 @@ snapshots:
picomatch: 4.0.3
webpack-virtual-modules: 0.6.2
unstorage@1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2):
unstorage@1.17.4(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0)))(ioredis@5.9.2):
dependencies:
anymatch: 3.1.3
chokidar: 5.0.0
@@ -31682,7 +31721,7 @@ snapshots:
ofetch: 1.5.1
ufo: 1.6.3
optionalDependencies:
db0: 0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.9)(pg@8.18.0)(sql.js@1.14.0))
db0: 0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.10)(pg@8.18.0)(sql.js@1.14.0))
ioredis: 5.9.2
untun@0.1.3:
+6 -3
View File
@@ -1,4 +1,4 @@
import { Pool } from 'pg';
import { Pool } from 'pg';
import { drizzle } from 'drizzle-orm/node-postgres';
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { eq, inArray, sql } from 'drizzle-orm';
@@ -110,7 +110,11 @@ async function rpcTransfer(args: Record<string, unknown>) {
const toId = Number(args.to_id ?? args.to);
const amount = Number(args.amount);
if (!Number.isInteger(fromId) || !Number.isInteger(toId) || !Number.isFinite(amount)) {
if (
!Number.isInteger(fromId) ||
!Number.isInteger(toId) ||
!Number.isFinite(amount)
) {
throw new Error('invalid transfer args');
}
if (fromId === toId || amount <= 0) return;
@@ -174,7 +178,6 @@ async function rpcGetAccount(args: Record<string, unknown>) {
};
}
async function rpcVerify() {
const rawInitial = process.env.SEED_INITIAL_BALANCE;
if (!rawInitial) {
@@ -1,4 +1,4 @@
import { query } from "./_generated/server";
import { query } from "./_generated/server";
import { v } from 'convex/values';
export const get_account = query(async ({ db }, { id }: { id: number }) => {
@@ -1,4 +1,4 @@
// convex/balances.ts
// convex/balances.ts
import { ShardedCounter } from '@convex-dev/sharded-counter';
import { components } from './_generated/api';
@@ -1,4 +1,4 @@
import { defineApp } from "convex/server";
import { defineApp } from "convex/server";
import shardedCounter from "@convex-dev/sharded-counter/convex.config.js";
const app = defineApp();
@@ -1,4 +1,4 @@
import { httpRouter } from "convex/server";
import { httpRouter } from "convex/server";
import { httpAction } from "./_generated/server";
import { api } from "./_generated/api";
import { seed_range } from './seed';
@@ -1,4 +1,4 @@
import { defineSchema, defineTable } from "convex/server";
import { defineSchema, defineTable } from "convex/server";
import { v } from "convex/values";
export default defineSchema({
@@ -1,4 +1,4 @@
import { v } from "convex/values";
import { v } from "convex/values";
import { mutation } from "./_generated/server";
export const clear_accounts = mutation({
@@ -1,4 +1,4 @@
import { mutation } from "./_generated/server";
import { mutation } from "./_generated/server";
import { v } from "convex/values";
import { accountBalances, accountKey } from './balances';
+1
View File
@@ -33,6 +33,7 @@
"@supabase/supabase-js": "^2.80.0",
"better-sqlite3": "^12.4.1",
"bun": "^1.3.2",
"cac": "^7.0.0",
"drizzle-orm": "^0.44.7",
"express": "^5.1.0",
"hdr-histogram-js": "^3.0.1",
@@ -24,7 +24,7 @@ const DURATION: &str = "5s";
const ALPHA: f32 = 1.5;
const CONNECTIONS: usize = 10;
const INIT_BALANCE: i64 = 1_000_000;
const AMOUNT: u32 = 1;
const AMOUNT: i64 = 1;
const ACCOUNTS: u32 = 100_000;
const CONFIRMED_READS: bool = true;
// Max inflight reducer calls imposed by the server.
@@ -57,7 +57,7 @@ async fn init_conn(cli: &Common, handle: &Handle) -> (JoinHandle<()>, Recv, Send
let params = WsParams {
compression: Compression::None,
light: true,
confirmed: cli.confirmed_reads.into(),
confirmed: cli.confirmed_reads.unwrap_or(CONFIRMED_READS).into(),
};
let conn = websocket::WsConnection::connect(uri, &cli.module, None, None, params)
@@ -130,7 +130,7 @@ fn bench(cli: &Common, bench: &Bench) {
// Dump some config parameters.
let alpha = bench.alpha;
let accounts = cli.accounts;
let amount = bench.amount as i64;
let amount = bench.amount;
if !cli.quiet {
println!("Benchmark parameters:");
println!("alpha={alpha}, amount = {amount}, accounts = {accounts}");
@@ -146,7 +146,7 @@ fn bench(cli: &Common, bench: &Bench) {
// Initialize connections.
let connections = bench.connections;
let confirmed_reads = cli.confirmed_reads;
let confirmed_reads = cli.confirmed_reads.unwrap_or(CONFIRMED_READS);
if !cli.quiet {
println!("initializing {connections} connections with confirmed-reads={confirmed_reads}");
}
@@ -261,8 +261,8 @@ struct Common {
#[arg(short, long, default_value = MODULE)]
module: String,
#[arg(long, default_value_t = CONFIRMED_READS)]
confirmed_reads: bool,
#[arg(long)]
confirmed_reads: Option<bool>,
#[arg(long, default_value_t = ACCOUNTS)]
accounts: u32,
@@ -292,7 +292,7 @@ struct Bench {
alpha: f32,
#[arg(long, default_value_t = AMOUNT)]
amount: u32,
amount: i64,
#[arg(short, long, default_value_t = CONNECTIONS)]
connections: usize,
+1 -1
View File
@@ -1,4 +1,4 @@
import 'dotenv/config';
import 'dotenv/config';
import { readdir, mkdir, writeFile } from 'node:fs/promises';
import { CONNECTORS } from './connectors';
import { runOne } from './core/runner';
+3 -4
View File
@@ -1,8 +1,7 @@
import type { RpcConnector } from '../core/connectors.ts';
import type { RpcConnector } from '../core/connectors.ts';
import { bunUrl } from '../opts.ts';
export default function bun(
url = process.env.BUN_URL || 'http://127.0.0.1:4000',
): RpcConnector {
export default function bun(url = bunUrl): RpcConnector {
if (!url) throw new Error('BUN_URL not set');
const baseUrl = url.replace(/\/+$/, '');
+3 -4
View File
@@ -1,8 +1,7 @@
import type { RpcConnector } from '../core/connectors.ts';
import type { RpcConnector } from '../core/connectors.ts';
import { convexUrl } from '../opts.ts';
export default function convex(
url = process.env.CONVEX_URL || 'http://127.0.0.1:3210',
): RpcConnector {
export default function convex(url = convexUrl): RpcConnector {
if (!url) throw new Error('CONVEX_URL not set');
function isWriteConflict(msg: unknown): boolean {
+2 -2
View File
@@ -6,6 +6,7 @@ import cockroach_rpc from './rpc/cockroach_rpc.ts';
import sqlite_rpc from './rpc/sqlite_rpc.ts';
import supabase_rpc from './rpc/supabase_rpc.ts';
import planetscale_pg_rpc from './rpc/planetscale_pg_rpc.ts';
import { ConnectorKey } from '../opts.ts';
export const CONNECTORS = {
convex,
@@ -17,5 +18,4 @@ export const CONNECTORS = {
sqlite_rpc,
supabase_rpc,
planetscale_pg_rpc,
};
export type ConnectorKey = keyof typeof CONNECTORS;
} satisfies Record<ConnectorKey, unknown>;
@@ -1,4 +1,4 @@
import type { RpcConnector } from '../../core/connectors.ts';
import type { RpcConnector } from '../../core/connectors.ts';
import { RpcRequest, RpcResponse } from './rpc_common.ts';
export default function cockroach_rpc(
@@ -1,4 +1,4 @@
import { RpcConnector } from '../../core/connectors.ts';
import { RpcConnector } from '../../core/connectors.ts';
import { RpcRequest, RpcResponse } from './rpc_common.ts';
export default function planetscale_pg_rpc(
@@ -1,4 +1,4 @@
import type { RpcConnector } from '../../core/connectors.ts';
import type { RpcConnector } from '../../core/connectors.ts';
import { RpcRequest, RpcResponse } from './rpc_common.ts';
export default function postgres_rpc(
@@ -1,4 +1,4 @@
export type RpcRequest = {
export type RpcRequest = {
name?: string;
args?: Record<string, unknown>;
};
@@ -1,32 +1,18 @@
import type { ReducerConnector } from '../core/connectors';
import * as mod from '../../module_bindings';
function useConfirmedReads(): boolean {
switch (process.env.STDB_CONFIRMED_READS) {
case '0':
return false;
case '1':
return true;
default:
return true;
}
}
import {
initialBalance,
stdbConfirmedReads,
stdbModule,
stdbUrl,
} from '../opts';
export function spacetimedb(
url = process.env.STDB_URL!,
moduleName = process.env.STDB_MODULE!,
url = stdbUrl,
moduleName = stdbModule,
): ReducerConnector {
let ready!: Promise<void>;
let ready: ReturnType<typeof Promise.withResolvers<void>>;
let conn: mod.DbConnection;
let resolveReady!: () => void;
let rejectReady!: (e: unknown) => void;
function armReady() {
ready = new Promise<void>((res, rej) => {
resolveReady = res;
rejectReady = rej;
});
}
async function connectWithBindings() {
if (!url) throw new Error('STDB_URL not set');
@@ -34,46 +20,48 @@ export function spacetimedb(
const Db = mod.DbConnection;
armReady();
ready = Promise.withResolvers<void>();
const subscriptions: string[] = [];
if (process.env.VERIFY === '1') {
console.log('[spacetimedb] subscribing to accounts');
subscriptions.push('SELECT * FROM accounts');
}
let subscribed = subscriptions.length === 0;
const subscribed = Promise.withResolvers<void>();
if (subscriptions.length === 0) subscribed.resolve();
const builder = Db.builder()
.withUri(url)
.withUri('ws://' + url)
.withDatabaseName(moduleName)
.withConfirmedReads(useConfirmedReads())
.withConfirmedReads(stdbConfirmedReads)
.onConnect((ctx) => {
console.log('[stdb] connected');
const conn = ctx;
resolveReady();
ready.resolve();
if (subscriptions.length > 0) {
conn
.subscriptionBuilder()
.onApplied((_sCtx) => {
subscribed = true;
subscribed.resolve();
})
.onError((ctx) => {
console.error('[stdb] subscription failed', ctx.event?.message);
subscribed.reject(ctx.event);
})
.subscribe(subscriptions);
}
})
.onConnectError((_ctx, err: any) => {
console.error('[stdb] onConnectError', err);
if (err instanceof Error) {
rejectReady(err);
ready.reject(err);
} else if (err && err.error instanceof Error) {
ready.reject(err.error);
} else if (err && typeof err.message === 'string') {
rejectReady(new Error(err.message));
ready.reject(new Error(err.message));
} else {
rejectReady(
ready.reject(
new Error(`Spacetime connection error: ${JSON.stringify(err)}`),
);
}
@@ -82,9 +70,8 @@ export function spacetimedb(
conn = builder.build();
while (!subscribed) {
await new Promise((res) => setTimeout(res, 25));
}
await ready.promise;
await subscribed.promise;
}
return {
@@ -94,9 +81,9 @@ export function spacetimedb(
async open() {
try {
await connectWithBindings();
await ready;
await ready.promise;
} catch (err) {
console.error('[spacetimedb] open() failed', err);
console.error('[spacetimedb] open() failed:', err);
throw err;
}
},
@@ -105,7 +92,7 @@ export function spacetimedb(
try {
conn.disconnect();
} catch (e) {
console.error('[spacetimedb] close() failed', e);
console.error('[spacetimedb] close() failed:', e);
}
},
@@ -122,13 +109,13 @@ export function spacetimedb(
},
async call(fn: string, args: Record<string, any>) {
await ready;
await ready.promise;
switch (fn) {
case 'seed': {
return conn.reducers.seed({
n: args.accounts,
initialBalance: args.initialBalance,
initialBalance: BigInt(args.initialBalance),
});
}
@@ -167,7 +154,7 @@ export function spacetimedb(
async verify() {
if (!conn) throw new Error('SpacetimeDB not connected');
const rawInitial = process.env.SEED_INITIAL_BALANCE;
const rawInitial = initialBalance;
if (!rawInitial) {
console.warn(
'[spacetimedb] SEED_INITIAL_BALANCE not set; skipping verification',
@@ -1,4 +1,4 @@
import type { Database as BetterSqlite3Database } from 'better-sqlite3';
import type { Database as BetterSqlite3Database } from 'better-sqlite3';
import path from 'path';
import fs from 'fs';
@@ -1,4 +1,4 @@
export interface CollisionStats {
export interface CollisionStats {
total: number; // total begin() calls
collisions: number; // how many times begin() found inflight > 0
collisionRate: number; // collisions / total
+1 -1
View File
@@ -1,4 +1,4 @@
export interface BaseConnector {
export interface BaseConnector {
name: string;
open(workers?: number): Promise<void>;
close(): Promise<void>;
+28 -38
View File
@@ -5,11 +5,22 @@ import { getSpacetimeCommittedTransfers } from './spacetimeMetrics.ts';
import { makeCollisionTracker } from './collision_tracker.ts';
import { RunResult } from './types.ts';
import { BaseConnector } from './connectors.ts';
import {
benchPipelined,
logErrors,
maxInflightPerWorker,
minOpTimeoutMs,
opTimeoutMs,
precomputedTransferPairs,
tailSlackMs,
useSpacetimeMetricsEndpoint,
verifyTransactions,
} from '../opts.ts';
const OP_TIMEOUT_MS = Number(process.env.BENCH_OP_TIMEOUT_MS ?? '15000');
const MIN_OP_TIMEOUT_MS = Number(process.env.MIN_OP_TIMEOUT_MS ?? '250');
const TAIL_SLACK_MS = Number(process.env.TAIL_SLACK_MS ?? '1000');
const DEFAULT_PRECOMPUTED_TRANSFER_PAIRS = 10_000_000;
const OP_TIMEOUT_MS = opTimeoutMs;
const MIN_OP_TIMEOUT_MS = minOpTimeoutMs;
const TAIL_SLACK_MS = tailSlackMs;
const PRECOMPUTED_TRANSFER_PAIRS = precomputedTransferPairs;
function precomputeZipfTransferPairs(
accounts: number,
@@ -101,8 +112,7 @@ export async function runOne({
}
const useSpacetimeMetrics =
process.env.USE_SPACETIME_METRICS_ENDPOINT === '1' &&
connector.name === 'spacetimedb';
useSpacetimeMetricsEndpoint && connector.name === 'spacetimedb';
let beforeTransfers: bigint | null = null;
if (useSpacetimeMetrics) {
@@ -126,13 +136,7 @@ export async function runOne({
}
}
const precomputedPairsRaw = Number(
process.env.BENCH_PRECOMPUTED_TRANSFER_PAIRS ??
DEFAULT_PRECOMPUTED_TRANSFER_PAIRS,
);
const precomputedPairs = Number.isFinite(precomputedPairsRaw)
? Math.max(1, Math.floor(precomputedPairsRaw))
: DEFAULT_PRECOMPUTED_TRANSFER_PAIRS;
const precomputedPairs = PRECOMPUTED_TRANSFER_PAIRS;
console.log(
`[${connector.name}] precomputing ${precomputedPairs} Zipf transfer pairs...`,
@@ -148,27 +152,13 @@ export async function runOne({
`[${connector.name}] precomputed ${transferPairs.count} pairs in ${(precomputeElapsedMs / 1000).toFixed(2)}s`,
);
const getEnvTernary = (envVal: string | undefined) => {
switch (envVal) {
case '0':
return false;
case '1':
return true;
default:
return null;
}
};
const PIPELINED =
getEnvTernary(process.env.BENCH_PIPELINED) ??
!!connector.maxInflightPerWorker;
const MAX_INFLIGHT_ENV = process.env.MAX_INFLIGHT_PER_WORKER;
const PIPELINED = benchPipelined ?? !!connector.maxInflightPerWorker;
const MAX_INFLIGHT_PER_WORKER =
MAX_INFLIGHT_ENV == null
maxInflightPerWorker === undefined
? (connector.maxInflightPerWorker ?? 8)
: MAX_INFLIGHT_ENV === '0'
: maxInflightPerWorker == 0
? Infinity
: Number(MAX_INFLIGHT_ENV);
: maxInflightPerWorker;
console.log(
`[${connector.name}] max inflight per worker: ${MAX_INFLIGHT_PER_WORKER}`,
@@ -240,7 +230,7 @@ export async function runOne({
);
ok = true;
} catch (err) {
if (process.env.LOG_ERRORS === '1') {
if (logErrors) {
const msg =
err instanceof Error
? `${err.name}: ${err.message}`
@@ -292,7 +282,7 @@ export async function runOne({
hist.recordValue(Math.max(1, Math.round((t1 - t0) * 1e3)));
}
} catch (err) {
if (process.env.LOG_ERRORS === '1') {
if (logErrors) {
const msg =
err instanceof Error
? `${err instanceof Error ? err.message : String(err)}`
@@ -383,10 +373,10 @@ export async function runOne({
return { start, completedWithinWindow, completedTotal, committedDelta };
};
const warmUpSeconds = 5;
console.log(`[${connector.name}] Warming up for ${warmUpSeconds}s...`);
await run(warmUpSeconds);
console.log(`[${connector.name}] Finished warmup.`);
// const warmUpSeconds = 5;
// console.log(`[${connector.name}] Warming up for ${warmUpSeconds}s...`);
// await run(warmUpSeconds);
// console.log(`[${connector.name}] Finished warmup.`);
console.log(`[${connector.name}] Starting workers for ${seconds}s run...`);
@@ -397,7 +387,7 @@ export async function runOne({
`[${connector.name}] All workers finished (including in-flight ops)`,
);
if (process.env.VERIFY === '1') {
if (verifyTransactions) {
console.log(`[${connector.name}] Running verification pass...`);
try {
await withOpTimeout(connector.verify(), `${connector.name} verify()`);
@@ -1,4 +1,6 @@
type LabelFilter = Record<string, string>;
import { stdbUrl } from '../opts';
type LabelFilter = Record<string, string>;
export async function fetchMetrics(url: string): Promise<string> {
const res = await fetch(url);
@@ -62,8 +64,7 @@ export function parseMetricCounter(
}
export async function getSpacetimeCommittedTransfers(): Promise<bigint | null> {
const url =
process.env.STDB_METRICS_URL ?? 'http://127.0.0.1:3000/v1/metrics';
const url = `http://${stdbUrl}/v1/metrics`;
const labels: LabelFilter = {
committed: 'true',
+1 -1
View File
@@ -1,4 +1,4 @@
export type RunResult = {
export type RunResult = {
tps: number;
samples: number;
committed_txns: number | null;
+1 -1
View File
@@ -1,4 +1,4 @@
/** Fast, deterministic PRNG (mulberry32). */
/** Fast, deterministic PRNG (mulberry32). */
function mulberry32(seed: number) {
let t = seed >>> 0;
return {
+48 -105
View File
@@ -1,94 +1,49 @@
import 'dotenv/config';
import { execSync } from 'node:child_process';
import { mkdir, writeFile } from 'node:fs/promises';
import { mkdir, writeFile, readFile } from 'node:fs/promises';
import { createConnection } from 'node:net';
import { join } from 'node:path';
import { ConnectorKey, CONNECTORS } from './connectors';
import { CONNECTORS } from './connectors';
import { runOne } from './core/runner';
import { initConvex } from './init/init_convex';
import { sh } from './init/utils';
import * as fs from 'fs';
import { setTimeout as sleep } from 'node:timers/promises';
import EventEmitter from 'node:events';
import {
accounts,
alpha,
concurrency,
ConnectorKey,
initialBalance,
noAnimation,
seconds,
skipPrep,
stdbConfirmedReads,
stdbModule,
stdbModulePath,
stdbUrl,
systems,
} from './opts';
// Simple TCP ping - just check if something is listening on the port
function ping(port: number, timeoutMs = 2000): Promise<boolean> {
return new Promise((resolve) => {
const socket = createConnection({ host: '127.0.0.1', port });
const timer = setTimeout(() => {
socket.destroy();
resolve(false);
}, timeoutMs);
socket.on('connect', () => {
clearTimeout(timer);
socket.destroy();
resolve(true);
});
socket.on('error', () => {
clearTimeout(timer);
resolve(false);
});
});
function ping(port: number, timeout = 2000): Promise<boolean> {
const socket = createConnection({ host: '127.0.0.1', port, timeout });
return EventEmitter.once(socket, 'connect').then(
() => true,
() => false,
);
}
// Use spacetime CLI to ping the server
function spacetimePing(): boolean {
async function spacetimePing(): Promise<boolean> {
try {
execSync('spacetime server ping local', { stdio: 'ignore' });
execSync('spacetime server ping ' + stdbUrl, { stdio: 'ignore' });
return true;
} catch {
return false;
}
}
// ============================================================================
// CLI Arguments
// ============================================================================
const args = process.argv.slice(2);
function getArg(name: string, defaultValue: number): number {
const idx = args.findIndex(
(a) => a === `--${name}` || a.startsWith(`--${name}=`),
);
if (idx === -1) return defaultValue;
const arg = args[idx];
if (arg.includes('=')) return Number(arg.split('=')[1]);
return Number(args[idx + 1] ?? defaultValue);
}
function getStringArg(name: string, defaultValue: string): string {
const idx = args.findIndex(
(a) => a === `--${name}` || a.startsWith(`--${name}=`),
);
if (idx === -1) return defaultValue;
const arg = args[idx];
if (arg.includes('=')) return arg.split('=')[1];
return args[idx + 1] ?? defaultValue;
}
function hasFlag(name: string): boolean {
return args.includes(`--${name}`);
}
const seconds = getArg('seconds', 10);
const concurrency = getArg('concurrency', 10);
const alpha = getArg('alpha', 1.5);
const systems = getStringArg('systems', 'convex,spacetimedb')
.split(',')
.map((s) => s.trim()) as ConnectorKey[];
const skipPrep = hasFlag('skip-prep');
const noAnimation = hasFlag('no-animation');
const accounts = Number(process.env.SEED_ACCOUNTS ?? 100_000);
const initialBalance = Number(process.env.SEED_INITIAL_BALANCE ?? 10_000_000);
// Force non-Docker mode and use metrics endpoint for TPS counting
process.env.USE_DOCKER = '0';
process.env.USE_SPACETIME_METRICS_ENDPOINT = '1';
// Set default SpacetimeDB config if not set
if (!process.env.STDB_URL) process.env.STDB_URL = 'ws://127.0.0.1:3000';
if (!process.env.STDB_MODULE) process.env.STDB_MODULE = 'test-1';
// ============================================================================
// ANSI Colors & Animation
// ============================================================================
@@ -136,10 +91,6 @@ function createSpinner(label: string): { stop: (finalText: string) => void } {
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// ============================================================================
// Service Health Checks
// ============================================================================
@@ -154,12 +105,12 @@ interface ServiceConfig {
const serviceConfigs: Record<string, ServiceConfig> = {
spacetimedb: {
name: 'SpacetimeDB',
healthCheck: async () => spacetimePing(),
healthCheck: spacetimePing,
startCmd: 'spacetime start',
},
spacetimedbRustClient: {
name: 'SpacetimeDB',
healthCheck: async () => spacetimePing(),
healthCheck: spacetimePing,
startCmd: 'spacetime start',
},
convex: {
@@ -239,41 +190,34 @@ async function prepSystem(system: ConnectorKey): Promise<void> {
try {
if (system === 'spacetimedb' || system == 'spacetimedbRustClient') {
const moduleName = process.env.STDB_MODULE || 'test-1';
const server = process.env.STDB_SERVER || 'local';
// const server2 = process.env.STDB_SERVER || 'http://localhost:3000';
const modulePath = process.env.STDB_MODULE_PATH || './spacetimedb';
// Publish module (creates DB if needed, updates if exists)
await sh('spacetime', [
'publish',
'-c',
'-y',
'--server',
server,
moduleName,
stdbUrl,
stdbModule,
'--module-path',
modulePath,
stdbModulePath,
]);
await sh('spacetime', [
'call',
'--server',
server,
moduleName,
'seed',
String(accounts),
String(initialBalance),
]);
console.log('[spacetimedb] seed complete.');
} else if (system === 'convex') {
}
if (system === 'convex') {
await initConvex();
} else {
const conn = connector();
await conn.open();
await conn.call('seed', { accounts, initialBalance });
await conn.close();
try {
await conn.call('seed', { accounts, initialBalance });
} finally {
await conn.close();
}
}
spinner.stop(c('green', '✓ READY'));
} catch (err: any) {
spinner.stop(c('red', `${err.message}`));
throw err;
}
}
@@ -315,9 +259,6 @@ async function runBenchmarkOther(
}
async function runBenchmarkStdb(): Promise<BenchResult | null> {
const moduleName = process.env.STDB_MODULE || 'test-1';
const server2 = process.env.STDB_SERVER || 'http://localhost:3000';
await sh('cargo', [
'run',
//"--quiet",
@@ -327,9 +268,9 @@ async function runBenchmarkStdb(): Promise<BenchResult | null> {
'bench',
//"--quiet",
'--server',
server2,
'ws://' + stdbUrl,
'--module',
moduleName,
stdbModule,
'--duration',
`${seconds}s`,
'--connections',
@@ -338,9 +279,11 @@ async function runBenchmarkStdb(): Promise<BenchResult | null> {
String(alpha),
'--tps-write-path',
'spacetimedb-tps.tmp.log',
'--confirmed-reads',
String(stdbConfirmedReads),
]);
const tpsStr = fs.readFileSync('spacetimedb-tps.tmp.log', 'utf-8').trim();
const tpsStr = (await readFile('spacetimedb-tps.tmp.log', 'utf-8')).trim();
const tps = Number(tpsStr);
if (isNaN(tps)) {
console.warn(`[spacetimedb] Failed to parse TPS from file: ${tpsStr}`);
+1 -1
View File
@@ -1,4 +1,4 @@
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { sqliteTable, integer as sqliteInt } from 'drizzle-orm/sqlite-core';
export const pgAccounts = pgTable('accounts', {
-24
View File
@@ -1,24 +0,0 @@
export function poolMax(
workers: number | undefined,
envName: string,
fallbackEnvMax: number,
defaultWorkers = 1,
): number {
const requested =
workers && Number.isFinite(workers) && workers > 0
? workers
: defaultWorkers;
const raw = process.env[envName];
const parsed = raw !== undefined ? Number(raw) : NaN;
const envMax =
Number.isFinite(parsed) && parsed > 0 ? parsed : fallbackEnvMax;
return Math.min(requested, envMax);
}
export function poolMaxFromEnv(fallback = 1000): number {
const raw = process.env.MAX_POOL;
const n = raw !== undefined ? Number(raw) : NaN;
return Number.isFinite(n) && n > 0 ? n : fallback;
}
+1 -1
View File
@@ -1,4 +1,4 @@
import 'dotenv/config';
import 'dotenv/config';
import { init_supabase } from './init_supabase.ts';
import { ACC, BAL, has, sh } from './utils.ts';
import { initSpacetime } from './init_spacetime.ts';
+1 -1
View File
@@ -1,4 +1,4 @@
import 'dotenv/config';
import 'dotenv/config';
export async function initBun(url?: string) {
const bunUrl = url ?? process.env.BUN_URL;
+5 -5
View File
@@ -1,4 +1,4 @@
const DEFAULT_CONVEX_URL = 'http://127.0.0.1:3210';
import { accounts, convexDir, convexUrl, initialBalance } from '../opts';
async function callConvexMutation(url: string, pathName: string, args: any) {
const res = await fetch(`${url}/api/mutation?format=json`, {
@@ -25,11 +25,11 @@ export async function initConvex() {
}
console.log('\n[convex] scaffold');
const dir = process.env.CONVEX_DIR || './convex-app';
const dir = convexDir;
const ACC = Number(process.env.SEED_ACCOUNTS ?? 100_000);
const BAL = Number(process.env.SEED_INITIAL_BALANCE ?? 1_000_000_000);
const url = process.env.CONVEX_URL || DEFAULT_CONVEX_URL;
const ACC = accounts;
const BAL = initialBalance;
const url = convexUrl;
console.log(
`[convex] expecting dev server at ${url} (start with: cd ${dir} && pnpm dev)`,
+1 -1
View File
@@ -1,4 +1,4 @@
import { ACC, BAL, waitFor } from './utils.ts';
import { ACC, BAL, waitFor } from './utils.ts';
import pg from 'pg';
export async function initPgLike(url: string, label: string) {
@@ -1,4 +1,4 @@
import { spawnServer } from './utils.ts';
import { spawnServer } from './utils.ts';
type RpcServerConfig = {
name: string;
@@ -1,5 +1,6 @@
import { join } from 'node:path';
import { ACC, BAL, sh } from './utils.ts';
import { stdbModule, stdbModulePath } from '../opts.ts';
export async function initSpacetime() {
const useMaincloud = process.env.STDB_MAINCLOUD === '1';
@@ -7,8 +8,8 @@ export async function initSpacetime() {
? 'maincloud'
: process.env.STDB_SERVER || 'local';
const dbName = process.env.STDB_MODULE!;
const modulePath = process.env.STDB_MODULE_PATH!;
const dbName = stdbModule;
const modulePath = stdbModulePath;
if (!dbName || !modulePath) {
console.log('[spacetimedb] missing STDB_MODULE/STDB_MODULE_PATH; skipping');
@@ -45,6 +46,7 @@ export async function initSpacetime() {
outDir,
'--module-path',
modulePath,
'-y',
]);
// 3) Seed
+1 -1
View File
@@ -1,4 +1,4 @@
import Database from 'better-sqlite3';
import Database from 'better-sqlite3';
import path from 'path';
import fs from 'fs';
import { ACC, BAL } from './utils.ts';
@@ -1,4 +1,4 @@
import pg, { Client as Pg } from 'pg';
import pg, { Client as Pg } from 'pg';
import { setTimeout as sleep } from 'node:timers/promises';
type PrepOpts = {
+5 -4
View File
@@ -1,9 +1,10 @@
import { spawn } from 'node:child_process';
import { spawn } from 'node:child_process';
import pg from 'pg';
import { setTimeout as sleep } from 'timers/promises';
import { setTimeout as sleep } from 'node:timers/promises';
import { accounts, initialBalance } from '../opts';
export const ACC = Number(process.env.SEED_ACCOUNTS ?? 100_000);
export const BAL = Number(process.env.SEED_INITIAL_BALANCE ?? 1_000_000);
export const ACC = accounts;
export const BAL = initialBalance;
export function has(v?: string) {
return v && v.trim().length > 0;
+268
View File
@@ -0,0 +1,268 @@
import cac from 'cac';
export const validConnectors = [
'convex',
'spacetimedb',
'spacetimedbRustClient',
'bun',
'postgres_rpc',
'cockroach_rpc',
'sqlite_rpc',
'supabase_rpc',
'planetscale_pg_rpc',
] as const;
export type ConnectorKey = (typeof validConnectors)[number];
interface OptionConfigBase {
env?: string;
}
interface OptionConfigNone extends OptionConfigBase {
type?: undefined;
}
interface OptionConfigString extends OptionConfigBase {
type: 'string';
default?: string;
}
interface OptionConfigNumber extends OptionConfigBase {
type: 'number';
default?: number;
}
interface OptionConfigBoolean extends OptionConfigBase {
type: 'boolean';
default?: boolean;
}
interface OptionConfigStrings extends OptionConfigBase {
type: 'strings';
possibleValues: readonly string[];
default?: readonly string[];
}
type OptionConfig =
| OptionConfigString
| OptionConfigNumber
| OptionConfigBoolean
| OptionConfigStrings
| OptionConfigNone;
class CLIParser {
constructor() {
this.cac.globalCommand.ignoreOptionDefaultValue();
this.cac.help().usage('[options]');
}
cac = cac();
#configs: Record<string, OptionConfig> = {};
option(
rawName: string,
description: string,
config: OptionConfig = {},
): this {
if (config.type === 'strings') {
description += ` (valid values: ${config.possibleValues.join(', ')})`;
}
if (config.env) {
description += ` [env: ${config.env}]`;
}
this.cac.option(rawName, description, {
default: 'default' in config ? config.default : undefined,
type: config.type === 'strings' ? [] : undefined,
});
const { name, isBoolean, negated } =
this.cac.globalCommand.options[this.cac.globalCommand.options.length - 1];
this.#configs[name] =
isBoolean && config.type === undefined
? { type: 'boolean', env: config.env, default: negated }
: config;
return this;
}
parse() {
const args = this.cac.parse();
this.cac.globalCommand.checkUnknownOptions();
this.cac.globalCommand.checkOptionValue();
this.cac.globalCommand.checkRequiredArgs();
this.cac.globalCommand.checkUnusedArgs();
const { options } = args;
if (options.help) {
process.exit(0);
}
for (const [name, config] of Object.entries(this.#configs)) {
if (config.env) options[name] ??= process.env[config.env];
let parser: (s: any) => any = (s) => s;
switch (config.type) {
case 'boolean':
parser = (s) =>
typeof s === 'boolean'
? s
: !(s === '0' || s === '' || s === 'false');
break;
case 'number':
parser = (s) => {
const n = Number(s);
if (Number.isFinite(n)) return n;
throw new Error(`invalid number '${s}'`);
};
break;
case 'strings':
if (options[name]?.length === 1 && options[name][0] === undefined) {
options[name] = undefined;
}
parser = (s: string | string[]) =>
(Array.isArray(s) ? s : s.split(',')).flat().map((s) => {
const x = s.trim();
if (!config.possibleValues.includes(x)) {
throw new Error(`${x} is not a valid value for this option`);
}
return x;
});
break;
}
if (options[name] !== undefined) {
options[name] = parser(options[name]);
} else if ('default' in config) {
options[name] = config.default;
}
}
return args;
}
}
const num = (defaultVal: number, env?: string): OptionConfig => ({
type: 'number',
default: defaultVal,
env,
});
const str = (defaultVal: string, env?: string): OptionConfig => ({
type: 'string',
default: defaultVal,
env,
});
const args = new CLIParser()
.option('--seconds <seconds>', 'Number of seconds to benchmark for', num(10))
.option('--concurrency <concurrency>', 'Concurrent clients to run', num(10))
.option('--alpha <alpha>', 'Alpha value', num(1.5))
.option('--systems <systems>', 'The systems to run against', {
type: 'strings',
possibleValues: validConnectors,
default: ['convex', 'spacetimedb'],
})
.option('--skip-prep', 'Skip prep')
.option('--no-animation', 'No animation')
.option(
'--accounts <num>',
'Number of accounts to run with',
num(100_000, 'SEED_ACCOUNTS'),
)
.option(
'--initial-balance <balance>',
'Initial balance for accounts',
num(10_000_000, 'SEED_INITIAL_BALANCE'),
)
.option(
'--stdb-url <url>',
'SpacetimeDB url',
str('127.0.0.1:3000', 'STDB_URL'),
)
.option(
'--stdb-module <name>',
'SpacetimeDB module name',
str('test-1', 'STDB_MODULE'),
)
.option(
'--stdb-module-path <dir>',
'SpacetimeDB module path',
str('./spacetimedb', 'STDB_MODULE_PATH'),
)
.option('--no-stdb-confirmed-reads', 'Disable confirmed reads', {
env: 'STDB_CONFIRMED_READS',
})
.option('--use-docker', 'Use docker', { env: 'USE_DOCKER' })
.option('--no-use-spacetime-metrics-endpoint', '', {
env: 'SPACETIME_METRICS_ENDPOINT',
})
.option(
'--pool-max <num>',
'Max pool size for postgres',
num(1000, 'MAX_POOL'),
)
.option(
'--bun-url <url>',
'Bun server url',
str('http://127.0.0.1:4000', 'BUN_URL'),
)
.option(
'--convex-url <url>',
'Convex server url',
str('http://127.0.0.1:3210', 'CONVEX_URL'),
)
.option(
'--convex-dir <dir>',
'Convex directory',
str('./convex-app', 'CONVEX_DIR'),
)
.option('--op-timeout-ms <num>', '', num(15000, 'BENCH_OP_TIMEOUT_MS'))
.option('--min-op-timeout-ms <num>', '', num(250, 'MIN_OP_TIMEOUT_MS'))
.option('--tail-slack-ms <num>', '', num(1000, 'TAIL_SLACK_MS'))
.option(
'--precomputed-transfer-pairs <num>',
'',
num(10_000_000, 'BENCH_PRECOMPUTED_TRANSFER_PAIRS'),
)
.option('--bench-pipelined', 'Force all systems to run pipelined', {
type: 'boolean',
env: 'BENCH_PIPELINED',
})
.option('--no-bench-pipelined', 'Disable request pipelining', {
type: 'boolean',
env: 'BENCH_PIPELINED',
})
.option(
'--max-inflight-per-worker <num>',
'When pipelining, max number of inflight requests allowed',
{ type: 'number', env: 'MAX_INFLIGHT_PER_WORKER' },
)
.option('--log-errors', 'Log errors', { env: 'LOG_ERRORS' })
.option('--verify-transactions', 'Verify transactions', { env: 'VERIFY' })
.parse();
const opts = args.options;
export const seconds: number = opts.seconds;
export const concurrency: number = opts.concurrency;
export const alpha: number = opts.alpha;
export const systems: ConnectorKey[] = opts.systems;
export const skipPrep: boolean = opts.skipPrep;
export const noAnimation: boolean = !opts.animation;
export const accounts: number = opts.accounts;
export const initialBalance: number = opts.initialBalance;
export const stdbUrl: string = opts.stdbUrl.replace(/^(http|ws)s?:\/\//, '');
export const stdbModule: string = opts.stdbModule;
export const stdbModulePath: string = opts.stdbModulePath;
export const stdbConfirmedReads: boolean = opts.stdbConfirmedReads;
export const useDocker: boolean = opts.useDocker;
export const useSpacetimeMetricsEndpoint: boolean =
opts.useSpacetimeMetricsEndpoint;
export const poolMax: number = opts.poolMax;
export const bunUrl: string = opts.bunUrl;
export const convexUrl: string = opts.convexUrl;
export const convexDir: string = opts.convexDir;
export const opTimeoutMs: number = opts.opTimeoutMs;
export const minOpTimeoutMs: number = opts.minOpTimeoutMs;
export const tailSlackMs: number = opts.tailSlackMs;
export const precomputedTransferPairs: number = opts.precomputedTransferPairs;
export const benchPipelined: boolean | undefined = opts.benchPipelined;
export const maxInflightPerWorker: number | undefined =
opts.maxInflightPerWorker;
export const logErrors: boolean = opts.logErrors;
export const verifyTransactions: boolean = opts.verifyTransactions;
@@ -1,11 +1,11 @@
import 'dotenv/config';
import 'dotenv/config';
import http from 'node:http';
import { Pool } from 'pg';
import { drizzle } from 'drizzle-orm/node-postgres';
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { eq, inArray, sql } from 'drizzle-orm';
import { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts';
import { poolMaxFromEnv } from '../helpers.ts';
import { poolMax } from '../opts.ts';
const CRDB_URL = process.env.CRDB_URL;
if (!CRDB_URL) {
@@ -20,7 +20,7 @@ const accounts = pgTable('accounts', {
const pool = new Pool({
connectionString: CRDB_URL,
application_name: 'crdb-rpc-drizzle',
max: poolMaxFromEnv(),
max: poolMax,
});
const db = drizzle(pool, { schema: { accounts } });
@@ -1,11 +1,11 @@
import 'dotenv/config';
import 'dotenv/config';
import http from 'node:http';
import { Pool } from 'pg';
import { drizzle } from 'drizzle-orm/node-postgres';
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { sql } from 'drizzle-orm';
import { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts';
import { poolMaxFromEnv } from '../helpers.ts';
import { poolMax } from '../opts.ts';
const PG_URL = process.env.PG_URL;
if (!PG_URL) {
@@ -20,7 +20,7 @@ const accounts = pgTable('accounts', {
const pool = new Pool({
connectionString: PG_URL,
application_name: 'pg-rpc-drizzle',
max: poolMaxFromEnv(),
max: poolMax,
});
const db = drizzle(pool, { schema: { accounts } });
@@ -1,11 +1,11 @@
import 'dotenv/config';
import 'dotenv/config';
import http from 'node:http';
import { Pool } from 'pg';
import { drizzle } from 'drizzle-orm/node-postgres';
import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core';
import { eq, inArray, sql } from 'drizzle-orm';
import type { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts';
import { poolMaxFromEnv } from '../helpers.ts';
import { poolMax } from '../opts.ts';
const DB_URL = process.env.SUPABASE_DB_URL ?? process.env.PG_URL;
if (!DB_URL) {
@@ -20,7 +20,7 @@ const accounts = pgTable('accounts', {
const pool = new Pool({
connectionString: DB_URL,
application_name: 'supabase-rpc-drizzle',
max: poolMaxFromEnv(),
max: poolMax,
});
const db = drizzle(pool, { schema: { accounts } });
@@ -1,4 +1,4 @@
import type { ReducerConnector } from '../core/connectors';
import type { ReducerConnector } from '../core/connectors';
export async function reducer_single(
conn: ReducerConnector,
@@ -1,4 +1,4 @@
import type { RpcConnector } from '../core/connectors';
import type { RpcConnector } from '../core/connectors';
export async function rpc_single_call(
conn: RpcConnector,
@@ -1,4 +1,4 @@
export async function sql_single_statement(
export async function sql_single_statement(
conn: unknown,
from: number,
to: number,
+1 -1
View File
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'bun',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'cockroach_rpc',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'convex',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'planetscale_pg_rpc',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'postgres_rpc',
@@ -1,4 +1,4 @@
import { reducer_single } from '../../scenario_recipes/reducer_single.ts';
import { reducer_single } from '../../scenario_recipes/reducer_single.ts';
export default {
system: 'spacetimedb',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'sqlite_rpc',
@@ -1,4 +1,4 @@
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
import { rpc_single_call } from '../../scenario_recipes/rpc_single_call.ts';
export default {
system: 'supabase_rpc',
+1 -1
View File
@@ -1,4 +1,4 @@
import type { ConnectorKey } from '../connectors';
import type { ConnectorKey } from '../opts';
export type TestCase = {
system: ConnectorKey;