on: pull_request: push: branches: - master merge_group: workflow_dispatch: inputs: pr_number: description: "Pull Request Number" required: false default: "" name: CI concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }} cancel-in-progress: true permissions: contents: read pull-requests: read jobs: merge_queue_noop: name: Detect merge queue no-op runs-on: ubuntu-latest outputs: skip: ${{ steps.compare.outputs.skip }} steps: - name: Checkout merge queue commit uses: actions/checkout@v4 - name: Compare merge queue commit to PR head id: compare env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} HEAD_REF: ${{ github.event.merge_group.head_ref }} shell: bash run: | set -euo pipefail echo "skip=false" >>"$GITHUB_OUTPUT" if [[ "${GITHUB_EVENT_NAME}" != "merge_group" ]]; then echo "Not a merge_group event; running CI normally." exit 0 fi if [[ ! "${HEAD_REF}" =~ /pr-([0-9]+)- ]]; then echo "Could not parse PR number from merge group head ref: ${HEAD_REF}" exit 0 fi pr_number="${BASH_REMATCH[1]}" pr_head_sha="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" --jq '.head.sha')" pr_head_repo="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" --jq '.head.repo.full_name')" if [[ -z "${pr_head_sha}" || "${pr_head_sha}" == "null" || -z "${pr_head_repo}" || "${pr_head_repo}" == "null" ]]; then echo "Could not resolve PR head SHA for PR #${pr_number}; running CI normally." exit 0 fi git fetch --no-tags --depth=1 "https://github.com/${pr_head_repo}.git" "${pr_head_sha}" if git diff --quiet "${pr_head_sha}" "${GITHUB_SHA}" --; then echo "Merge queue commit ${GITHUB_SHA} has the same tree as PR #${pr_number} head ${pr_head_sha}; skipping duplicate CI." echo "skip=true" >>"$GITHUB_OUTPUT" else echo "Merge queue commit ${GITHUB_SHA} differs from PR #${pr_number} head ${pr_head_sha}; running CI normally." fi smoketests: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Smoketests (${{ matrix.name }}) strategy: matrix: include: - name: Linux runner: spacetimedb-new-runner-2 - name: Windows runner: spacetimedb-windows-runner runs-on: ${{ matrix.runner }} timeout-minutes: 120 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb cache-on-failure: false cache-all-crates: true cache-workspace-crates: true prefix-key: v1 - uses: actions/setup-dotnet@v4 with: global-json-file: global.json # nodejs and pnpm are required for the typescript quickstart smoketest - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 18 - uses: ./.github/actions/setup-pnpm with: run_install: true # Install emscripten for C++ module compilation tests. - name: Install emscripten (Linux) if: runner.os == 'Linux' shell: bash run: | git clone https://github.com/emscripten-core/emsdk.git ~/emsdk cd ~/emsdk ./emsdk install 4.0.21 ./emsdk activate 4.0.21 - name: Install emscripten (Windows) if: runner.os == 'Windows' shell: pwsh run: | git clone https://github.com/emscripten-core/emsdk.git $env:USERPROFILE\emsdk cd $env:USERPROFILE\emsdk .\emsdk install 4.0.21 .\emsdk activate 4.0.21 - name: Install psql (Windows) if: runner.os == 'Windows' shell: pwsh run: | # Fail properly if any individual command fails $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true choco install psql -y --no-progress # Check for existence, since `choco` doesn't seem to fail the step if it fails to install.. # See https://github.com/clockworklabs/SpacetimeDB/pull/4399 for more background. Get-Command psql - name: Update dotnet workloads if: runner.os == 'Windows' run: | # Fail properly if any individual command fails $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them dotnet workload config --update-mode manifests dotnet workload update - name: Override NuGet packages shell: bash run: | dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime dotnet pack -c Release crates/bindings-csharp/Runtime cd sdks/csharp ./tools~/write-nuget-config.sh ../.. # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # ChatGPT suspects that this could be due to different build invocations using the same target dir, # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # `cargo build --manifest-path` (which apparently build different dependency trees). # However, we've been unable to fix it so... /shrug - name: Check v8 outputs shell: bash run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean --release -p v8 || true cargo build --release -p v8 fi - name: Install cargo-nextest uses: taiki-e/install-action@nextest # --test-threads=1 eliminates contention in the C# tests where they fight over bindings # build artifacts. # It also seemed to improve performance a fair amount (11m -> 6m) - name: Run smoketests (Linux) if: runner.os == 'Linux' shell: bash run: | if [ -f ~/emsdk/emsdk_env.sh ]; then source ~/emsdk/emsdk_env.sh fi cargo ci smoketests -- --test-threads=1 # Due to Emscripten PATH issues this was separated to make sure OpenSSL still builds correctly - name: Run smoketests (Windows) if: runner.os == 'Windows' shell: pwsh run: | if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") { & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null } cargo ci smoketests -- --test-threads=1 # this is a no-op version of the above check with a trivially-passing body. # we can't just let the check be entirely skipped because each matrix target is a required check, # and skipping this check means that the matrix isn't "populated" so the required checks wouldn't be met. smoketests_noop: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip == 'true' }} name: Smoketests (${{ matrix.name }}) strategy: matrix: include: - name: Linux - name: Windows runs-on: ubuntu-latest steps: - name: Skip duplicate merge queue smoketest run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." test: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Test Suite runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the smoketests job save the cache since it builds the most things save-if: false prefix-key: v1 - name: Check v8 outputs shell: bash run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean --release -p v8 || true cargo build --release -p v8 fi - uses: actions/setup-dotnet@v3 with: global-json-file: global.json - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 22 - uses: ./.github/actions/setup-pnpm with: run_install: true # Install cmake and emscripten for C++ module compilation tests. - name: Install cmake and emscripten run: | sudo apt-get update sudo apt-get install -y cmake git clone https://github.com/emscripten-core/emsdk.git ~/emsdk cd ~/emsdk ./emsdk install 4.0.21 ./emsdk activate 4.0.21 - name: Install wasm-bindgen CLI run: | REQUIRED_WASM_BINDGEN_VERSION="$( awk ' $1 == "name" && $3 == "\"wasm-bindgen\"" { in_pkg = 1; next } in_pkg && $1 == "version" { gsub(/"/, "", $3); print $3; exit; } ' Cargo.lock )" if [ -z "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then echo "Failed to determine wasm-bindgen version from Cargo.lock" exit 1 fi INSTALLED_WASM_BINDGEN_VERSION="$(wasm-bindgen --version 2>/dev/null | awk '{print $2}' || true)" if [ "${INSTALLED_WASM_BINDGEN_VERSION}" != "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" fi wasm-bindgen --version - name: Check engine simulation build run: cargo check -p spacetimedb-engine --no-default-features --features simulation # Source emsdk environment to make emcc (Emscripten compiler) available in PATH. - name: Run tests run: | source ~/emsdk/emsdk_env.sh cargo ci test keynote_bench: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Keynote Bench runs-on: spacetimedb-benchmark-runner concurrency: group: ci-benchmark-runner queue: max timeout-minutes: 60 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} # Node 24 is the current Active LTS line. - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 24 - uses: ./.github/actions/setup-pnpm with: package_json_file: package.json working_directory: . run_install: "true" - uses: ./.github/actions/keynote-bench-setup with: public_root: . rust_cache_workspaces: ${{ github.workspace }} - name: Run keynote-2 benchmark regression check run: cargo ci keynote-bench index_scan_bench: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Index Scan Bench runs-on: spacetimedb-benchmark-runner concurrency: group: ci-benchmark-runner queue: max timeout-minutes: 60 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb save-if: false prefix-key: v1 - name: Run index scan benchmark regression check run: cargo bench -p spacetimedb-bench --bench index_scan_gate lints: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Lints runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Checkout sources uses: actions/checkout@v3 - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - run: echo ::add-matcher::.github/workflows/rust_matcher.json - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the smoketests job save the cache since it builds the most things save-if: false prefix-key: v1 - uses: actions/setup-dotnet@v3 with: global-json-file: global.json - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 22 - uses: ./.github/actions/setup-pnpm with: run_install: true - name: Run ci lint run: cargo ci lint wasm_bindings: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Build and test wasm bindings runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - uses: actions/checkout@v3 - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - run: echo ::add-matcher::.github/workflows/rust_matcher.json - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the smoketests job save the cache since it builds the most things save-if: false prefix-key: v1 - name: Run bindgen tests run: cargo ci wasm-bindings publish_checks: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check that packages are publishable runs-on: spacetimedb-new-runner-2 permissions: read-all env: RUST_BACKTRACE: full steps: - uses: actions/checkout@v3 - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Run checks run: cargo ci publish-checks update: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Test spacetimedb-update flow (${{ matrix.target }}) permissions: read-all strategy: matrix: include: - { target: x86_64-unknown-linux-gnu, runner: spacetimedb-new-runner-2 } - { target: aarch64-apple-darwin, runner: macos-latest } - { target: x86_64-pc-windows-msvc, runner: windows-latest } runs-on: ${{ matrix.runner }} env: RUST_BACKTRACE: full steps: - name: Checkout uses: actions/checkout@v3 - name: Install Rust uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Install rust target run: rustup target add ${{ matrix.target }} - name: Test spacetimedb-update env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth # this is a no-op version of the above check with a trivially-passing body. # we can't just let the check be entirely skipped because each matrix target is a required check, # and skipping this check means that the matrix isn't "populated" so the required checks wouldn't be met. update_noop: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip == 'true' }} name: Test spacetimedb-update flow (${{ matrix.target }}) permissions: read-all strategy: matrix: include: - { target: x86_64-unknown-linux-gnu } - { target: aarch64-apple-darwin } - { target: x86_64-pc-windows-msvc } runs-on: ubuntu-latest steps: - name: Skip duplicate merge queue update test run: echo "Merge queue commit has the same tree as the PR head; update-flow already ran for the PR." unreal_engine_tests: name: Unreal Engine Tests # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use # a custom runner. runs-on: spacetimedb-new-runner-2 # Disable the tests because they are very flaky at the moment. # TODO: Remove this line and re-enable the `if` line just below here. if: false # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway. # if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} container: image: ghcr.io/epicgames/unreal-engine:dev-5.6 credentials: # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and # it seems to be totally happy. # However, the token needs to be for a user that has access to the EpicGames org (see # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6) username: ${{ github.actor }} password: ${{ secrets.GHCR_TOKEN }} # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle. options: --user 0:0 env: RUST_BACKTRACE: full steps: # Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back # commits though. # - name: Find Git ref # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # shell: bash # run: | # PR_NUMBER="${{ github.event.inputs.pr_number || null }}" # if test -n "${PR_NUMBER}"; then # GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" # else # GIT_REF="${{ github.ref }}" # fi # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Run Unreal Engine tests working-directory: sdks/unreal env: UE_ROOT_PATH: /home/ue4/UnrealEngine run: | apt-get update apt-get install -y acl curl ca-certificates REPO="$GITHUB_WORKSPACE" # Let ue4 read/write the workspace & tool caches without changing ownership for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true done # Rust tool caches live under the runner tool cache so they persist export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo" export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup" mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME" # Make sure the UE build script is executable (and parents traversable) UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}" chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true # Run the build & tests as ue4 (who owns the UE tree) sudo -E -H -u ue4 env \ HOME=/home/ue4 \ XDG_CONFIG_HOME=/home/ue4/.config \ CARGO_HOME="$CARGO_HOME" \ RUSTUP_HOME="$RUSTUP_HOME" \ PATH="$CARGO_HOME/bin:$PATH" \ bash -lc ' set -euxo pipefail # Install rustup for ue4 if needed (uses the shared caches) if ! command -v cargo >/dev/null 2>&1; then curl -sSf https://sh.rustup.rs | sh -s -- -y fi rustup show >/dev/null git config --global --add safe.directory "$GITHUB_WORKSPACE" || true cd "$GITHUB_WORKSPACE/sdks/unreal" cargo --version cargo test -- --test-threads=1 ' ci_command_docs: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check CI command docs runs-on: spacetimedb-new-runner-2 env: RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Check for docs change run: cargo ci self-docs --check cli_docs: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check CLI docs permissions: read-all runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 22 - uses: ./.github/actions/setup-pnpm with: run_install: true - name: Get pnpm store directory shell: bash run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - uses: actions/cache@v4 name: Setup pnpm cache with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the smoketests job save the cache since it builds the most things save-if: false prefix-key: v1 - name: Check for docs change run: | cargo ci cli-docs unity-testsuite: needs: [merge_queue_noop, lints] # Skip if this is an external contribution. # The license secrets will be empty, so the step would fail anyway. if: ${{ needs.merge_queue_noop.outputs.skip != 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} permissions: contents: read checks: write runs-on: spacetimedb-unity-runner timeout-minutes: 30 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Checkout repository id: checkout-stdb uses: actions/checkout@v4 # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests. - name: Setup dotnet uses: actions/setup-dotnet@v3 with: global-json-file: global.json - name: Override NuGet packages run: | dotnet pack crates/bindings-csharp/BSATN.Runtime dotnet pack crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. # This means that (if version numbers match) we will test the local versions of the C# packages, even # if they're not pushed to NuGet. # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - name: Restore .NET solution working-directory: sdks/csharp run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln # Now, setup the Unity tests. - name: Patch spacetimedb dependency in Cargo.toml working-directory: demo/Blackholio/server-rust run: | sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml cat Cargo.toml - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the main CI job save the cache since it builds the most things save-if: false prefix-key: v1 # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # ChatGPT suspects that this could be due to different build invocations using the same target dir, # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # `cargo build --manifest-path` (which apparently build different dependency trees). # However, we've been unable to fix it so... /shrug - name: Check v8 outputs run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean --release -p v8 || true cargo build --release -p v8 fi - name: Install SpacetimeDB CLI from the local checkout run: | export CARGO_HOME="$HOME/.cargo" echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" cargo install --force --path crates/cli --locked --message-format=short cargo install --force --path crates/standalone --locked --message-format=short # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - name: Generate client bindings working-directory: demo/Blackholio/server-rust run: bash ./generate.sh -y - name: Check for changes run: | tools/check-diff.sh demo/Blackholio/client-unity/Assets/Scripts/autogen || { echo 'Error: Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.' exit 1 } - name: Hydrate Unity SDK DLLs run: cargo regen csharp dlls - name: Check Unity meta files uses: DeNA/unity-meta-check@v3 with: enable_pr_comment: ${{ github.event_name == 'pull_request' }} target_path: sdks/csharp env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: Start SpacetimeDB run: | spacetime start & disown - name: Publish unity-tests module to SpacetimeDB working-directory: demo/Blackholio/server-rust run: | spacetime logout && spacetime login --server-issued-login local bash ./publish.sh - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json working-directory: demo/Blackholio/client-unity/Packages run: | yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json cat manifest.json - uses: actions/cache@v3 with: path: demo/Blackholio/client-unity/Library key: Unity-${{ github.head_ref }} restore-keys: Unity- - name: Run Unity tests uses: game-ci/unity-test-runner@v4 with: unityVersion: 2022.3.32f1 # Adjust Unity version to a valid tag projectPath: demo/Blackholio/client-unity # Path to the Unity project subdirectory githubToken: ${{ secrets.GITHUB_TOKEN }} testMode: playmode useHostNetwork: true artifactsPath: "" env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} godot-testsuite: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} permissions: contents: read runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target UseLocalBsatnRuntime: true steps: - name: Checkout repository id: checkout-stdb uses: actions/checkout@v4 - name: Setup dotnet uses: actions/setup-dotnet@v3 with: global-json-file: global.json - name: Override NuGet packages run: | dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime dotnet pack -c Release crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. # This means that (if version numbers match) we will test the local versions of the C# packages, even # if they're not pushed to NuGet. # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - name: Restore .NET solution working-directory: sdks/csharp run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln # Now, setup the Godot tests. - name: Patch spacetimedb dependency in Cargo.toml working-directory: demo/Blackholio/server-rust run: | sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml cat Cargo.toml - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the main CI job save the cache since it builds the most things save-if: false prefix-key: v1 # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # ChatGPT suspects that this could be due to different build invocations using the same target dir, # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # `cargo build --manifest-path` (which apparently build different dependency trees). # However, we've been unable to fix it so... /shrug - name: Check v8 outputs run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean --release -p v8 || true cargo build --release -p v8 fi - name: Install SpacetimeDB CLI from the local checkout run: | export CARGO_HOME="$HOME/.cargo" echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" cargo install --force --path crates/cli --locked --message-format=short cargo install --force --path crates/standalone --locked --message-format=short # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - name: Generate client bindings working-directory: demo/Blackholio/server-rust run: bash ./generate.sh -y - name: Check for changes run: | tools/check-diff.sh demo/Blackholio/client-godot/module_bindings || { echo 'Error: Godot bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.' exit 1 } - name: Patch SpacetimeDB Godot SDK dependency working-directory: demo/Blackholio/client-godot run: | dotnet remove package SpacetimeDB.ClientSDK.Godot dotnet add reference ../../../sdks/csharp/SpacetimeDB.ClientSDK.Godot.csproj cat blackholio.csproj - name: Setup Godot uses: chickensoft-games/setup-godot@v2 with: version: 4.6.2 use-dotnet: true - name: Restore Godot project working-directory: demo/Blackholio/client-godot run: dotnet restore --configfile ../../../NuGet.Config blackholio.csproj - name: Build Godot project run: godot --headless --verbose --path demo/Blackholio/client-godot --build-solutions --quit - name: Start SpacetimeDB run: | spacetime start & disown - name: Publish godot-tests module to SpacetimeDB working-directory: demo/Blackholio/server-rust run: | spacetime login --server-issued-login local bash ./publish.sh - name: Run Godot tests run: godot --headless --path demo/Blackholio/client-godot --scene res://tests/GodotPlayModeTests.tscn csharp-testsuite: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} runs-on: spacetimedb-new-runner-2 timeout-minutes: 30 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Checkout repository id: checkout-stdb uses: actions/checkout@v4 - name: Setup dotnet uses: actions/setup-dotnet@v3 with: global-json-file: global.json - name: Override NuGet packages run: | dotnet pack crates/bindings-csharp/BSATN.Runtime dotnet pack crates/bindings-csharp/Runtime # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. # This means that (if version numbers match) we will test the local versions of the C# packages, even # if they're not pushed to NuGet. # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - name: Restore .NET solution working-directory: sdks/csharp run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln - name: Run .NET tests working-directory: sdks/csharp run: dotnet test -warnaserror --no-restore SpacetimeDB.ClientSDK.csproj - name: Verify C# formatting working-directory: sdks/csharp run: dotnet format --no-restore --verify-no-changes SpacetimeDB.ClientSDK.sln - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the main CI job save the cache since it builds the most things save-if: false prefix-key: v1 # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # ChatGPT suspects that this could be due to different build invocations using the same target dir, # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # `cargo build --manifest-path` (which apparently build different dependency trees). # However, we've been unable to fix it so... /shrug - name: Check v8 outputs run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean -p v8 || true cargo build -p v8 fi find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean --release -p v8 || true cargo build --release -p v8 fi - name: Install SpacetimeDB CLI from the local checkout run: | export CARGO_HOME="$HOME/.cargo" echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" cargo install --force --path crates/cli --locked --message-format=short cargo install --force --path crates/standalone --features allow_loopback_http_for_tests --locked --message-format=short # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - name: Check quickstart-chat bindings are up to date run: | bash sdks/csharp/tools~/gen-quickstart.sh tools/check-diff.sh templates/chat-console-cs/module_bindings || { echo 'Error: quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.' exit 1 } # TODO: Re-enable this once csharp is using the v2 ws api. # - name: Check client-api bindings are up to date # working-directory: sdks/csharp # run: | # bash tools~/gen-client-api.sh # "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/SpacetimeDB/ClientApi || { # echo 'Error: Client API bindings are dirty. Please run `sdks/csharp/tools~/gen-client-api.sh`.' # exit 1 # } - name: Start SpacetimeDB run: | spacetime start & disown - name: Run regression tests run: | bash sdks/csharp/tools~/run-regression-tests.sh tools/check-diff.sh sdks/csharp/examples~/regression-tests || { echo 'Error: Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.' exit 1 } global_json_policy: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Verify global.json files are symlinks runs-on: ubuntu-latest permissions: contents: read env: RUST_BACKTRACE: full steps: - name: Find Git ref env: PR_NUMBER: ${{ github.event.inputs.pr_number }} run: | if [ -n "$PR_NUMBER" ]; then GIT_REF="refs/pull/$PR_NUMBER/merge" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb save-if: false prefix-key: v1 - name: Check global.json policy run: cargo ci global-json-policy smoketests_mod_rs_complete: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Check smoketests/mod.rs is complete runs-on: ubuntu-latest permissions: contents: read env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full steps: - name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | PR_NUMBER="${{ github.event.inputs.pr_number || null }}" if test -n "${PR_NUMBER}"; then GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" else GIT_REF="${{ github.ref }}" fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb cache-on-failure: false cache-all-crates: true cache-workspace-crates: true prefix-key: v1 # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # ChatGPT suspects that this could be due to different build invocations using the same target dir, # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # `cargo build --manifest-path` (which apparently build different dependency trees). # However, we've been unable to fix it so... /shrug - name: Check v8 outputs shell: bash run: | find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then echo "Could not find v8 output file librusty_v8.a; rebuilding manually." cargo clean -p v8 || true cargo build -p v8 fi - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries run: | cargo ci smoketests check-mod-list docs-build: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Docs build runs-on: spacetimedb-new-runner-2 env: RUST_BACKTRACE: full steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '22' - uses: ./.github/actions/setup-pnpm with: run_install: true - name: Get pnpm store directory working-directory: sdks/typescript shell: bash run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - uses: actions/cache@v4 name: Setup pnpm cache with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Docusaurus build run: cargo ci docs typescript-test: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: TypeScript - Tests runs-on: spacetimedb-new-runner-2 env: RUST_BACKTRACE: full steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 22 - uses: ./.github/actions/setup-pnpm with: run_install: true - name: Get pnpm store directory shell: bash working-directory: crates/bindings-typescript run: | echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - uses: actions/cache@v4 name: Setup pnpm cache with: path: ${{ env.STORE_PATH }} key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- # - name: Extract SpacetimeDB branch name from file # id: extract-branch # run: | # # Define the path to the branch file # BRANCH_FILE=".github/spacetimedb-branch.txt" # # Default to master if file doesn't exist # if [ ! -f "$BRANCH_FILE" ]; then # echo "::notice::No SpacetimeDB branch file found, using 'master'" # echo "branch=master" >> $GITHUB_OUTPUT # exit 0 # fi # # Read and trim whitespace from the file # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') # # Fallback to master if empty # if [ -z "$branch" ]; then # echo "::warning::SpacetimeDB branch file is empty, using 'master'" # branch="master" # fi # echo "branch=$branch" >> $GITHUB_OUTPUT # echo "Using SpacetimeDB branch from file: $branch" - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} shared-key: spacetimedb # Let the main CI job save the cache since it builds the most things save-if: false prefix-key: v1 # # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. # # ChatGPT suspects that this could be due to different build invocations using the same target dir, # # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with # # `cargo build --manifest-path` (which apparently build different dependency trees). # # However, we've been unable to fix it so... /shrug # - name: Check v8 outputs # run: | # find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true # if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then # echo "Could not find v8 output file librusty_v8.a; rebuilding manually." # cargo clean -p v8 || true # cargo build -p v8 # fi # if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then # echo "Could not find v8 output file librusty_v8.a; rebuilding manually." # cargo clean --release -p v8 || true # cargo build --release -p v8 # fi # - name: Install SpacetimeDB CLI from the local checkout # run: | # export CARGO_HOME="$HOME/.cargo" # echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" # cargo install --force --path crates/cli --locked --message-format=short # cargo install --force --path crates/standalone --locked --message-format=short # # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). # ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime # # Clear any existing information # spacetime server clear -y - name: Run TypeScript tests run: cargo ci typescript-test # - name: Run quickstart-chat tests # working-directory: examples/quickstart-chat # run: pnpm test # # # Run this step always, even if the previous steps fail # - name: Print rows in the user table # if: always() # run: spacetime sql quickstart-chat "SELECT * FROM user"