From 7a2a697ee459a4e1efd905e9adf8f9fa9f7dcf1f Mon Sep 17 00:00:00 2001
From: bradleyshep <148254416+bradleyshep@users.noreply.github.com>
Date: Fri, 5 Jun 2026 09:01:21 -0400
Subject: [PATCH] CI - fix LLM benchmark workflows (#5181)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

# Description of Changes

Updates `llm-benchmark-periodic.yml` and
`llm-benchmark-validate-goldens.yml` to new CI infrastructure.

- Switch to the current runner (`spacetimedb-new-runner-2`)
- Drop the dead `localhost:5000` container + `--privileged`
- Build the local SpacetimeDB server the benchmark harness needs, and
use that
  same local CLI for publishing

# API and ABI breaking changes

None. CI-only.

# Expected complexity level and risk

1 — workflow-only, no production code.

# Testing

- [ ] Run both workflows via `workflow_dispatch` and confirm they pass
---
 .github/workflows/llm-benchmark-periodic.yml   | 18 ++++++++----------
 .../llm-benchmark-validate-goldens.yml         | 18 ++++++++----------
 2 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml
index f8b4c1fb17..40ad2c75fe 100644
--- a/.github/workflows/llm-benchmark-periodic.yml
+++ b/.github/workflows/llm-benchmark-periodic.yml
@@ -29,19 +29,10 @@ concurrency:
 
 jobs:
   run-benchmarks:
-    runs-on: spacetimedb-new-runner
-    container:
-      image: localhost:5000/spacetimedb-ci:latest
-      options: >-
-        --privileged
+    runs-on: spacetimedb-new-runner-2
     timeout-minutes: 180
 
     steps:
-      - name: Install spacetime CLI
-        run: |
-          curl -sSf https://install.spacetimedb.com | sh -s -- -y
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
       - name: Checkout master
         uses: actions/checkout@v4
         with:
@@ -75,6 +66,13 @@ jobs:
       - name: Build llm-benchmark tool
         run: cargo install --path tools/xtask-llm-benchmark --locked
 
+      - name: Build SpacetimeDB server for benchmark harness
+        run: |
+          cargo ci smoketests prepare
+          mkdir -p "$HOME/.local/bin"
+          ln -sf "$GITHUB_WORKSPACE/target/release/spacetimedb-cli" "$HOME/.local/bin/spacetime"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Run benchmarks
         env:
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
diff --git a/.github/workflows/llm-benchmark-validate-goldens.yml b/.github/workflows/llm-benchmark-validate-goldens.yml
index f9d634c50e..17384a654e 100644
--- a/.github/workflows/llm-benchmark-validate-goldens.yml
+++ b/.github/workflows/llm-benchmark-validate-goldens.yml
@@ -15,11 +15,7 @@ concurrency:
 
 jobs:
   validate-goldens:
-    runs-on: spacetimedb-new-runner
-    container:
-      image: localhost:5000/spacetimedb-ci:latest
-      options: >-
-        --privileged
+    runs-on: spacetimedb-new-runner-2
     timeout-minutes: 60
 
     strategy:
@@ -28,11 +24,6 @@ jobs:
         lang: [rust, csharp, typescript]
 
     steps:
-      - name: Install spacetime CLI
-        run: |
-          curl -sSf https://install.spacetimedb.com | sh -s -- -y
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
       - name: Checkout master
         uses: actions/checkout@v4
         with:
@@ -70,6 +61,13 @@ jobs:
       - name: Build llm-benchmark tool
         run: cargo install --path tools/xtask-llm-benchmark --locked
 
+      - name: Build SpacetimeDB server for benchmark harness
+        run: |
+          cargo ci smoketests prepare
+          mkdir -p "$HOME/.local/bin"
+          ln -sf "$GITHUB_WORKSPACE/target/release/spacetimedb-cli" "$HOME/.local/bin/spacetime"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
       - name: Validate golden answers (${{ matrix.lang }})
         env:
           MSBUILDDISABLENODEREUSE: "1"