mirror of
https://github.com/clockworklabs/SpacetimeDB.git
synced 2026-05-09 17:19:45 -04:00
bradleyshep
efa6f382b1
# Description of Changes LLM benchmark updates for local development: - **Local SDK paths**: Templates use relative paths to workspace crates (`crates/bindings`, `crates/bindings-csharp`, `crates/bindings-typescript`) instead of published packages, so the bench runs against local SDK changes. - **NODEJS_DIR support**: On Windows (e.g. nvm4w), if `pnpm` is not on PATH, the bench uses `NODEJS_DIR` to locate `pnpm` and prepends it to PATH for subprocesses. - **Refactor**: Extracted `relative_to_workspace()` in `templates.rs` and removed noisy `NODEJS_DIR` logging in `publishers.rs`. - **Benchmark results**: Updated `docs/llms/llm-comparison-details.json` and `docs/llms/llm-comparison-summary.json`. # API and ABI breaking changes None. # Expected complexity level and risk **2** — Local-only changes to the benchmark tool. Templates now require local SDKs to be built (especially TypeScript: `pnpm build` in `crates/bindings-typescript`). No impact on published SDKs or runtime. # Testing - [ ] Run `cargo llm run --lang rust --modes docs --providers openai` from repo root - [ ] Run TypeScript benchmarks with `pnpm build` in `crates/bindings-typescript` first - [ ] On Windows with nvm4w, set `NODEJS_DIR` if `pnpm` is not on PATH and run TypeScript benchmarks --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: clockwork-labs-bot <bot@clockworklabs.com> Co-authored-by: clockwork-labs-bot <clockwork-labs-bot@users.noreply.github.com>
40 lines
851 B
TOML
40 lines
851 B
TOML
[package]
|
|
name = "xtask-llm-benchmark"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
default-run = "llm_benchmark"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
spacetimedb.workspace = true
|
|
spacetimedb-guard.workspace = true
|
|
spacetimedb-data-structures.workspace = true
|
|
|
|
anyhow.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
blake3.workspace = true
|
|
clap.workspace = true
|
|
chrono = { version = "0.4", features = ["clock", "serde"] }
|
|
dotenvy = "0.15"
|
|
async-trait = "0.1.89"
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
|
|
urlencoding = "2.1.3"
|
|
reqwest = { version = "0.12", features = ["json"] }
|
|
futures = "0.3.31"
|
|
regex = "1"
|
|
tempfile = "3.23.0"
|
|
fs2 = "0.4.3"
|
|
heck = "0.5.0"
|
|
thiserror = "2.0.17"
|
|
|
|
[lib]
|
|
path = "src/lib.rs"
|
|
|
|
[[bin]]
|
|
name = "llm_benchmark"
|
|
path = "src/bin/llm_benchmark.rs"
|