CI - Cache more of our build outputs (#3521)

# Description of Changes

Changes with cache hit:
CI / Smoketests (spacetimedb-runner): 17m -> 10m
CI / Smoketests (windows-latest): 39m -> 31m
CI / Lints: 3m30s -> 2m
unity-testsuite: 23m -> 19m

This included creating a stripped-down `Dockerfile` that only runs
SpacetimeDB (as opposed to e.g. building tracy, flamegraphs, etc.). That
shaves off about 4-5m on the linux smoketests job.

# API and ABI breaking changes

No. CI only.

# Expected complexity level and risk

2

# Testing

Ran existing CI and noted the change in times in the description above

---------

Signed-off-by: Zeke Foppa <196249+bfops@users.noreply.github.com>
Co-authored-by: Zeke Foppa <bfops@users.noreply.github.com>
Co-authored-by: John Detter <4099508+jdetter@users.noreply.github.com>

.github/Dockerfile

@@ -0,0 +1,6 @@
+# Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker
+FROM rust:1.90.0
+RUN mkdir -p /stdb/data
+COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli /usr/local/bin/
+COPY ./crates/standalone/config.toml /stdb/data/config.toml
+RUN ln -s /usr/local/bin/spacetimedb-cli /usr/local/bin/spacetime

.github/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  node:
+    labels:
+      app: spacetimedb
+    build:
+      context: ../
+      dockerfile: .github/Dockerfile
+    ports:
+      - "3000:3000"
+      # Postgres
+      - "5432:5432"
+    entrypoint: spacetime start --pg-port 5432
+    privileged: true
+    environment:
+      RUST_BACKTRACE: 1

.github/workflows/ci.yml

@@ -34,7 +34,8 @@ jobs:
             container: null
     runs-on: ${{ matrix.runner }}
     container: ${{ matrix.container }}
-    
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -53,6 +54,15 @@ jobs:
         with:
           ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: true
+          cache-all-crates: true
+          cache-workspace-crates: true
+
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
@@ -71,13 +81,18 @@ jobs:
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+      - name: Build crates
+        run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
       - name: Start Docker daemon
         if: runner.os == 'Linux'
         run: /usr/local/bin/start-docker.sh
 
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
-        run: docker compose up -d
+        run: |
+          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
+          rm .dockerignore
+          docker compose -f .github/docker-compose.yml up -d
       - name: Build and start database (Windows)
         if: runner.os == 'Windows'
         run: |
@@ -85,7 +100,6 @@ jobs:
           $ErrorActionPreference = 'Stop'
           $PSNativeCommandUseErrorActionPreference = $true
 
-          cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
           Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
           cd modules
           # the sdk-manifests on windows-latest are messed up, so we need to update them
@@ -101,7 +115,7 @@ jobs:
         run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
-        run: docker compose down
+        run: docker compose -f .github/docker-compose.yml down
 
   test:
     name: Test Suite
@@ -110,6 +124,8 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -129,6 +145,13 @@ jobs:
           ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
 
       - uses: actions/setup-dotnet@v3
         with:
@@ -179,6 +202,8 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -186,6 +211,14 @@ jobs:
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - uses: actions/setup-dotnet@v3
         with:
           global-json-file: global.json
@@ -222,12 +255,22 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - uses: actions/checkout@v3
 
       - uses: dsherret/rust-toolchain-file@v1
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Run bindgen tests
         run: cargo test -p spacetimedb-codegen
 
@@ -408,6 +451,8 @@ jobs:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
@@ -451,6 +496,14 @@ jobs:
 
       - uses: dsherret/rust-toolchain-file@v1
 
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+
       - name: Check for docs change
         run: |
           cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md

.github/workflows/csharp-test.yml

@@ -23,6 +23,8 @@ jobs:
       group: unity-test-${{ github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
     timeout-minutes: 30
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout repository
         id: checkout-stdb
@@ -70,26 +72,18 @@ jobs:
 
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
-        id: cache-rust-deps
         with:
-          workspaces: demo/Blackholio/server-rust
-          key: ${{ steps.checkout-stdb.outputs.commit }}
-          # Cache Rust deps even if unit tests have failed.
-          cache-on-failure: true
-          # Cache the CLI as well.
-          cache-all-crates: true
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
 
       - name: Install SpacetimeDB CLI from the local checkout
-        # Rebuild only if we didn't get a precise cache hit.
-        if: steps.cache-rust-deps.outputs.cache-hit == 'false'
         run: |
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
           ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
-        env:
-          # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice.
-          CARGO_TARGET_DIR: demo/Blackholio/server-rust/target
 
       - name: Check quickstart-chat bindings are up to date
         working-directory: sdks/csharp

.github/workflows/typescript-test.yml

@@ -14,6 +14,8 @@ concurrency:
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
     steps:
       - name: Checkout repository
@@ -81,10 +83,10 @@ jobs:
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
-          workspaces: |
-            .
-            modules/quickstart-chat
-          shared-key: quickstart-chat-test
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the main CI job save the cache since it builds the most things
+          save-if: false
 
       - name: Install SpacetimeDB CLI from the local checkout
         run: |

smoketests/__init__.py

@@ -48,7 +48,7 @@ USE_SPACETIME_LOGIN = False
 REMOTE_SERVER = False
 
 # default value can be overridden by `--compose-file` flag
-COMPOSE_FILE = "./docker-compose.yml"
+COMPOSE_FILE = ".github/docker-compose.yml"
 
 # this will be initialized by main()
 STDB_CONFIG = ''