on:
  pull_request:
  push:
    branches:
      - master
  merge_group:
  workflow_dispatch:
    inputs:
      pr_number:
        description: "Pull Request Number"
        required: false
        default: ""

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr_number || format('sha-{0}', github.sha) }}
  cancel-in-progress: true

jobs:
  smoketests:
    name: Smoketests
    strategy:
      fail-fast: false
      matrix:
        runner: [spacetimedb-new-runner, windows-latest]
        include:
          - runner: spacetimedb-new-runner
            container:
              image: localhost:5000/spacetimedb-ci:latest
              options: --privileged
          - runner: windows-latest
            container: null
    runs-on: ${{ matrix.runner }}
    container: ${{ matrix.container }}
    timeout-minutes: 120
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}
      - uses: dsherret/rust-toolchain-file@v1
      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          cache-on-failure: false
          cache-all-crates: true
          cache-workspace-crates: true
          prefix-key: v1

      - uses: actions/setup-dotnet@v4
        with:
          global-json-file: global.json

      # nodejs and pnpm are required for the typescript quickstart smoketest
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Install psql (Windows)
        if: runner.os == 'Windows'
        run: choco install psql -y --no-progress
        shell: powershell

      - name: Update dotnet workloads
        if: runner.os == 'Windows'
        run: |
          # Fail properly if any individual command fails
          $ErrorActionPreference = 'Stop'
          $PSNativeCommandUseErrorActionPreference = $true

          cd modules
          # the sdk-manifests on windows-latest are messed up, so we need to update them
          dotnet workload config --update-mode manifests
          dotnet workload update

      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
      # `cargo build --manifest-path` (which apparently build different dependency trees).
      # However, we've been unable to fix it so... /shrug
      - name: Check v8 outputs
        shell: bash
        run: |
          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
            cargo clean -p v8 || true
            cargo build -p v8
          fi

      - name: Install cargo-nextest
        uses: taiki-e/install-action@nextest

      - name: Run smoketests
        run: cargo ci smoketests

  smoketests-python:
    needs: [lints]
    name: Smoketests (Python Legacy)
    strategy:
      matrix:
        runner: [spacetimedb-new-runner, windows-latest]
        include:
          - runner: spacetimedb-new-runner
            smoketest_args: --docker
            container:
              image: localhost:5000/spacetimedb-ci:latest
              options: --privileged
          - runner: windows-latest
            smoketest_args: --no-build-cli
            container: null
    runs-on: ${{ matrix.runner }}
    container: ${{ matrix.container }}
    timeout-minutes: 120
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}
      - uses: dsherret/rust-toolchain-file@v1
      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          cache-on-failure: false
          cache-all-crates: true
          cache-workspace-crates: true
          prefix-key: v1

      - uses: actions/setup-dotnet@v4
        with:
          global-json-file: global.json

      - name: Override NuGet packages
        shell: bash
        run: |
          dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
          dotnet pack -c Release crates/bindings-csharp/Runtime
          cd sdks/csharp
          ./tools~/write-nuget-config.sh ../..

      # nodejs and pnpm are required for the typescript quickstart smoketest
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Install psql (Windows)
        if: runner.os == 'Windows'
        run: choco install psql -y --no-progress
        shell: powershell

      - name: Build crates
        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update

      - name: Start Docker daemon
        if: runner.os == 'Linux'
        run: /usr/local/bin/start-docker.sh

      - name: Build and start database (Linux)
        if: runner.os == 'Linux'
        run: |
          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
          rm .dockerignore
          docker compose -f .github/docker-compose.yml up -d

      - name: Build and start database (Windows)
        if: runner.os == 'Windows'
        run: |
          # Fail properly if any individual command fails
          $ErrorActionPreference = 'Stop'
          $PSNativeCommandUseErrorActionPreference = $true

          Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
          cd modules
          # the sdk-manifests on windows-latest are messed up, so we need to update them
          dotnet workload config --update-mode manifests
          dotnet workload update

      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
        if: runner.os == 'Windows'

      - name: Install python deps
        run: python -m pip install -r smoketests/requirements.txt

      - name: Run Python smoketests
        # Note: clear_database and replication only work in private
        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams

      - name: Stop containers (Linux)
        if: always() && runner.os == 'Linux'
        run: docker compose -f .github/docker-compose.yml down

  test:
    needs: [lints]
    name: Test Suite
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}

      - uses: dsherret/rust-toolchain-file@v1
      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the smoketests job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Build typescript module sdk
        working-directory: crates/bindings-typescript
        run: pnpm build

      - name: Run tests
        run: cargo ci test

  lints:
    name: Lints
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - uses: dsherret/rust-toolchain-file@v1
      - run: echo ::add-matcher::.github/workflows/rust_matcher.json

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the smoketests job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Run ci lint
        run: cargo ci lint

  wasm_bindings:
    name: Build and test wasm bindings
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - uses: actions/checkout@v3

      - uses: dsherret/rust-toolchain-file@v1
      - run: echo ::add-matcher::.github/workflows/rust_matcher.json

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the smoketests job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - name: Run bindgen tests
        run: cargo ci wasm-bindings

  publish_checks:
    name: Check that packages are publishable
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    permissions: read-all
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python env
        run: |
          test -d venv || python3 -m venv venv
          venv/bin/pip3 install argparse toml
      - name: Run checks
        run: |
          set -ueo pipefail
          FAILED=0
          ROOTS=(spacetimedb spacetimedb-sdk)
          CRATES=$(venv/bin/python3 tools/find-publish-list.py --recursive --directories --quiet "${ROOTS[@]}")
          for crate_dir in $CRATES; do
            if ! venv/bin/python3 tools/crate-publish-checks.py "${crate_dir}"; then
              FAILED=$(( $FAILED + 1 ))
            fi
          done
          if [ $FAILED -gt 0 ]; then
            exit 1
          fi

  update:
    name: Test spacetimedb-update flow (${{ matrix.target }})
    permissions: read-all
    strategy:
      matrix:
        include:
          - runner: spacetimedb-new-runner
            target: x86_64-unknown-linux-gnu
            container:
              image: localhost:5000/spacetimedb-ci:latest
              options: --privileged
          - { target: aarch64-unknown-linux-gnu, runner: arm-runner }
          - { target: aarch64-apple-darwin, runner: macos-latest }
          - { target: x86_64-pc-windows-msvc, runner: windows-latest }
    runs-on: ${{ matrix.runner }}
    container: ${{ matrix.container }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install Rust
        uses: dsherret/rust-toolchain-file@v1

      - name: Install rust target
        run: rustup target add ${{ matrix.target }}

      - name: Install packages
        if: ${{ matrix.runner == 'arm-runner' }}
        shell: bash
        run: sudo apt install -y libssl-dev

      - name: Build spacetimedb-update
        run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update
        if: runner.os == 'Windows'

      - name: Run self-install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          ROOT_DIR="$(mktemp -d)"
          # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
          # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
          # happens very frequently on the `macos-runner`, but we haven't seen it on any others).
          cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
          "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
        if: runner.os == 'Windows'

      - name: Test spacetimedb-update
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth
        if: runner.os != 'Windows'

  unreal_engine_tests:
    name: Unreal Engine Tests
    # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use
    # a custom runner.
    runs-on: spacetimedb-new-runner
    # Disable the tests because they are very flaky at the moment.
    # TODO: Remove this line and re-enable the `if` line just below here.
    if: false
    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
    # if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
    container:
      image: ghcr.io/epicgames/unreal-engine:dev-5.6
      credentials:
        # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and
        # it seems to be totally happy.
        # However, the token needs to be for a user that has access to the EpicGames org (see
        # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6)
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_TOKEN }}
      # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run
      # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle.
      options: --user 0:0
    steps:
      # Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back
      # commits though.
      #      - name: Find Git ref
      #        env:
      #          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #        shell: bash
      #        run: |
      #          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
      #          if test -n "${PR_NUMBER}"; then
      #            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
      #          else
      #            GIT_REF="${{ github.ref }}"
      #          fi
      #          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}
      - uses: dsherret/rust-toolchain-file@v1
      - name: Run Unreal Engine tests
        working-directory: sdks/unreal
        env:
          UE_ROOT_PATH: /home/ue4/UnrealEngine
        run: |

          apt-get update
          apt-get install -y acl curl ca-certificates

          REPO="$GITHUB_WORKSPACE"
          # Let ue4 read/write the workspace & tool caches without changing ownership
          for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do
            [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true
          done

          # Rust tool caches live under the runner tool cache so they persist
          export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo"
          export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup"
          mkdir -p "$CARGO_HOME" "$RUSTUP_HOME"
          chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME"

          # Make sure the UE build script is executable (and parents traversable)
          UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}"
          chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true
          chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true

          # Run the build & tests as ue4 (who owns the UE tree)
          sudo -E -H -u ue4 env \
            HOME=/home/ue4 \
            XDG_CONFIG_HOME=/home/ue4/.config \
            CARGO_HOME="$CARGO_HOME" \
            RUSTUP_HOME="$RUSTUP_HOME" \
            PATH="$CARGO_HOME/bin:$PATH" \
            bash -lc '
              set -euxo pipefail
              # Install rustup for ue4 if needed (uses the shared caches)
              if ! command -v cargo >/dev/null 2>&1; then
                curl -sSf https://sh.rustup.rs | sh -s -- -y
              fi
              rustup show >/dev/null
              git config --global --add safe.directory "$GITHUB_WORKSPACE" || true

              cd "$GITHUB_WORKSPACE/sdks/unreal"
              cargo --version
              cargo test -- --test-threads=1
            '

  ci_command_docs:
    name: Check CI command docs
    runs-on: ubuntu-latest
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}

      - uses: dsherret/rust-toolchain-file@v1

      - name: Check for docs change
        run: cargo ci self-docs --check

  cli_docs:
    name: Check CLI docs
    permissions: read-all
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Find Git ref
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
          if test -n "${PR_NUMBER}"; then
            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
          else
            GIT_REF="${{ github.ref }}"
          fi
          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.GIT_REF }}

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        with:
          run_install: true

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - uses: actions/cache@v4
        name: Setup pnpm cache
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - uses: dsherret/rust-toolchain-file@v1

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the smoketests job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - name: Check for docs change
        run: |
          cargo ci cli-docs

  llm_ci_check:
    name: Verify LLM benchmark is up to date
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      # Build the tool from master to ensure consistent hash computation
      # with the llm-benchmark-update workflow (which also uses master's tool).
      - name: Checkout master (build tool from trusted code)
        uses: actions/checkout@v4
        with:
          ref: master
          fetch-depth: 1

      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Install llm-benchmark tool from master
        run: |
          cargo install --path tools/xtask-llm-benchmark --locked
          command -v llm_benchmark

      # Now checkout the PR branch to verify its benchmark files
      - name: Checkout PR branch
        uses: actions/checkout@v4
        with:
          clean: false

      - name: Run hash check (both langs)
        run: llm_benchmark ci-check

  unity-testsuite:
    needs: [lints]
    # Skip if this is an external contribution.
    # The license secrets will be empty, so the step would fail anyway.
    if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
    permissions:
      contents: read
      checks: write
    runs-on: spacetimedb-unity-runner
    timeout-minutes: 30
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Checkout repository
        id: checkout-stdb
        uses: actions/checkout@v4

      # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests.

      - name: Setup dotnet
        uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Override NuGet packages
        run: |
          dotnet pack crates/bindings-csharp/BSATN.Runtime
          dotnet pack crates/bindings-csharp/Runtime

          # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository
          # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if
          # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages.
          # This means that (if version numbers match) we will test the local versions of the C# packages, even
          # if they're not pushed to NuGet.
          # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file.
          cd sdks/csharp
          ./tools~/write-nuget-config.sh ../..

      - name: Restore .NET solution
        working-directory: sdks/csharp
        run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln

      # Now, setup the Unity tests.
      - name: Patch spacetimedb dependency in Cargo.toml
        working-directory: demo/Blackholio/server-rust
        run: |
          sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml
          cat Cargo.toml

      - name: Install Rust toolchain
        uses: dsherret/rust-toolchain-file@v1

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the main CI job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - name: Install SpacetimeDB CLI from the local checkout
        run: |
          cargo install --force --path crates/cli --locked --message-format=short
          cargo install --force --path crates/standalone --locked --message-format=short
          # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
          ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime

      - name: Generate client bindings
        working-directory: demo/Blackholio/server-rust
        run: bash ./generate.sh -y

      - name: Check for changes
        run: |
          tools/check-diff.sh demo/Blackholio/client-unity/Assets/Scripts/autogen || {
              echo 'Error: Bindings are dirty. Please run `demo/Blackholio/server-rust/generate.sh`.'
              exit 1
          }

      - name: Hydrate Unity SDK DLLs
        run: cargo ci dlls

      - name: Check Unity meta files
        uses: DeNA/unity-meta-check@v3
        with:
          enable_pr_comment: ${{ github.event_name == 'pull_request' }}
          target_path: sdks/csharp
        env:
          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

      - name: Start SpacetimeDB
        run: |
          spacetime start &
          disown

      - name: Publish unity-tests module to SpacetimeDB
        working-directory: demo/Blackholio/server-rust
        run: |
          spacetime logout && spacetime login --server-issued-login local
          bash ./publish.sh

      - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json
        working-directory: demo/Blackholio/client-unity/Packages
        run: |
          yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json
          cat manifest.json

      - uses: actions/cache@v3
        with:
          path: demo/Blackholio/client-unity/Library
          key: Unity-${{ github.head_ref }}
          restore-keys: Unity-

      - name: Run Unity tests
        uses: game-ci/unity-test-runner@v4
        with:
          unityVersion: 2022.3.32f1 # Adjust Unity version to a valid tag
          projectPath: demo/Blackholio/client-unity # Path to the Unity project subdirectory
          githubToken: ${{ secrets.GITHUB_TOKEN }}
          testMode: playmode
          useHostNetwork: true
          artifactsPath: ""
        env:
          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
          UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}

  csharp-testsuite:
    needs: [lints]
    runs-on: spacetimedb-new-runner
    container:
      image: localhost:5000/spacetimedb-ci:latest
      options: >-
        --privileged
        --cgroupns=host
    timeout-minutes: 30
    env:
      CARGO_TARGET_DIR: ${{ github.workspace }}/target
    steps:
      - name: Checkout repository
        id: checkout-stdb
        uses: actions/checkout@v4

      # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests.

      - name: Setup dotnet
        uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json

      - name: Override NuGet packages
        run: |
          dotnet pack crates/bindings-csharp/BSATN.Runtime
          dotnet pack crates/bindings-csharp/Runtime

          # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository
          # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if
          # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages.
          # This means that (if version numbers match) we will test the local versions of the C# packages, even
          # if they're not pushed to NuGet.
          # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file.
          cd sdks/csharp
          ./tools~/write-nuget-config.sh ../..

      - name: Restore .NET solution
        working-directory: sdks/csharp
        run: dotnet restore --configfile NuGet.Config SpacetimeDB.ClientSDK.sln

      - name: Run .NET tests
        working-directory: sdks/csharp
        run: dotnet test -warnaserror --no-restore

      - name: Verify C# formatting
        working-directory: sdks/csharp
        run: dotnet format --no-restore --verify-no-changes SpacetimeDB.ClientSDK.sln

      - name: Install Rust toolchain
        uses: dsherret/rust-toolchain-file@v1

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ github.workspace }}
          shared-key: spacetimedb
          # Let the main CI job save the cache since it builds the most things
          save-if: false
          prefix-key: v1

      - name: Install SpacetimeDB CLI from the local checkout
        run: |
          cargo install --force --path crates/cli --locked --message-format=short
          cargo install --force --path crates/standalone --locked --message-format=short
          # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
          ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime

      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
      # `cargo build --manifest-path` (which apparently build different dependency trees).
      # However, we've been unable to fix it so... /shrug
      - name: Check v8 outputs
        run: |
          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
            cargo clean -p v8 || true
            cargo build -p v8
          fi

      - name: Check quickstart-chat bindings are up to date
        working-directory: sdks/csharp
        run: |
          bash tools~/gen-quickstart.sh
          "${GITHUB_WORKSPACE}"/tools/check-diff.sh examples~/quickstart-chat || {
              echo 'Error: quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.'
              exit 1
          }

      - name: Check client-api bindings are up to date
        working-directory: sdks/csharp
        run: |
          bash tools~/gen-client-api.sh
          "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/SpacetimeDB/ClientApi || {
              echo 'Error: Client API bindings are dirty. Please run `sdks/csharp/tools~/gen-client-api.sh`.'
              exit 1
          }

      - name: Start SpacetimeDB
        run: |
          spacetime start &
          disown

      - name: Run regression tests
        run: |
          bash sdks/csharp/tools~/run-regression-tests.sh
          tools/check-diff.sh sdks/csharp/examples~/regression-tests || {
              echo 'Error: Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.'
              exit 1
          }

  internal-tests:
    name: Internal Tests
    needs: [lints]
    # Skip if not a PR or a push to master
    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
    if: ${{ (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master'))
      && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
    permissions:
      contents: read
    runs-on: ubuntu-latest
    env:
      TARGET_OWNER: clockworklabs
      TARGET_REPO: SpacetimeDBPrivate
    steps:
      - id: dispatch
        name: Trigger tests
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
          script: |
            const workflowId = 'ci.yml';
            const targetRef = 'master';
            const targetOwner = process.env.TARGET_OWNER;
            const targetRepo = process.env.TARGET_REPO;
            // Use the ref for pull requests because the head sha is brittle (github does some extra dance where it merges in master).
            const publicRef = (context.eventName === 'pull_request') ? context.payload.pull_request.head.ref : context.sha;
            const preDispatch = new Date().toISOString();
  
            // Dispatch the workflow in the target repository
            await github.rest.actions.createWorkflowDispatch({
              owner: targetOwner,
              repo: targetRepo,
              workflow_id: workflowId,
              ref: targetRef,
              inputs: { public_ref: publicRef }
            });
  
            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
  
            // Find the dispatched run by name
            let runId = null;
            for (let attempt = 0; attempt < 20 && !runId; attempt++) { // up to ~10 minutes to locate the run
              await sleep(5000);
              const runsResp = await github.rest.actions.listWorkflowRuns({
                owner: targetOwner,
                repo: targetRepo,
                workflow_id: workflowId,
                event: 'workflow_dispatch',
                branch: targetRef,
                per_page: 50,
              });
  
              const expectedName = `CI [public_ref=${publicRef}]`;
              const candidates = runsResp.data.workflow_runs
                .filter(r => r.name === expectedName && new Date(r.created_at) >= new Date(preDispatch))
                .sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
  
              if (candidates.length > 0) {
                runId = candidates[0].id;
                break;
              }
            }
  
            if (!runId) {
              core.setFailed('Failed to locate dispatched run in the private repository.');
              return;
            }
  
            const runUrl = `https://github.com/${targetOwner}/${targetRepo}/actions/runs/${runId}`;
            core.info(`View run: ${runUrl}`);
            core.setOutput('run_id', String(runId));
            core.setOutput('run_url', runUrl);
  
      - name: Wait for Internal Tests to complete
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
          script: |
            const targetOwner = process.env.TARGET_OWNER;
            const targetRepo = process.env.TARGET_REPO;
            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
            const runUrl = `${{ steps.dispatch.outputs.run_url }}`;
            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
  
            core.info(`Waiting for workflow result... ${runUrl}`);
  
            let conclusion = null;
            for (let attempt = 0; attempt < 240; attempt++) { // up to ~2 hours
              const runResp = await github.rest.actions.getWorkflowRun({
                owner: targetOwner,
                repo: targetRepo,
                run_id: runId
              });
              const { status, conclusion: c } = runResp.data;
              if (status === 'completed') {
                conclusion = c || 'success';
                break;
              }
              await sleep(30000);
            }
  
            if (!conclusion) {
              core.setFailed('Timed out waiting for private workflow to complete.');
              return;
            }
  
            if (conclusion !== 'success') {
              core.setFailed(`Private workflow failed with conclusion: ${conclusion}`);
            }
  
      - name: Cancel invoked run if workflow cancelled
        if: ${{ cancelled() }}
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
          script: |
            const targetOwner = process.env.TARGET_OWNER;
            const targetRepo = process.env.TARGET_REPO;
            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
            if (!runId) return;
            await github.rest.actions.cancelWorkflowRun({
              owner: targetOwner,
              repo: targetRepo,
              run_id: runId,
            });

  warn-python-smoketests:
    name: Check for Python smoketest edits
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    permissions:
      contents: read
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Fail if Python smoketests were modified
        run: |
          PYTHON_SMOKETEST_CHANGES=$(git diff --name-only origin/${{ github.base_ref }} HEAD -- 'smoketests/**.py')

          if [ -n "$PYTHON_SMOKETEST_CHANGES" ]; then
            echo "::error::This PR modifies legacy Python smoketests. Please add new tests to the Rust smoketests in crates/smoketests/ instead."
            echo ""
            echo "Changed files:"
            echo "$PYTHON_SMOKETEST_CHANGES"
            echo ""
            echo "The Python smoketests are being replaced by Rust smoketests."
            echo "See crates/smoketests/DEVELOP.md for instructions on adding Rust smoketests."
            exit 1
          fi

          echo "No Python smoketest changes detected."