SpacetimeDB/.github/workflows/benchmarks.yml

on:
  push:
    branches:
      - master
      - jgilles/fix-callgrind-again

  workflow_dispatch:
    inputs:
      pr_number:
        description: 'Pull Request Number'
        required: false
        default: ''

  issue_comment:
    types: [created]

name: Benchmarks

env:
  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  GH_REPO: ${{ github.repository }}

jobs:
  benchmark:
    name: run criterion benchmarks
    runs-on: benchmarks-runner
    # filter for a comment containing 'benchmarks please'
    if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
    env:
      PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
    steps:
      - name: Clear stdb dir
        if: always()
        run: |
          rm -fr /stdb/*

      - name: Enable CPU boost
        run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost

      - name: Check membership
        if: ${{ github.event_name == 'issue_comment' }}
        env:
          CONTRIB_ORG: clockworklabs
          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
          ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
        run: |
          curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
          if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
            gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
            exit 1
          fi

      - name: Post initial comment
        run: |
          if [[ $PR_NUMBER ]]; then
            comment_parent=issues/$PR_NUMBER
            comment_update=issues/comments
          else
            comment_parent=commits/$GITHUB_SHA
            comment_update=comments
          fi
          comment_body="Benchmark in progress..."
          comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
          echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV

      - name: find PR branch
        if: ${{ env.PR_NUMBER }}
        run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ env.PR_REF || github.ref }}
          # if we're on master we want to know what the sha of HEAD~1 is so
          # that we can compare results from it to HEAD (in the "Fetch markdown
          # summary PR" step). otherwise, we can use a fully shallow checkout
          fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}

      - name: Install Rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          components: clippy
          toolchain: stable
          target: wasm32-unknown-unknown
          override: true

      - name: Install .NET toolchain
        uses: actions/setup-dotnet@v3
        with:
          global-json-file: global.json
        env:
          DOTNET_INSTALL_DIR: ~/.dotnet

      - name: Build
        working-directory: crates/bench/
        run: |
          cargo build --release

      - name: Install latest wasm-opt for module optimisations
        run: |
          curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1

      - name: Disable CPU boost
        run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost

      - name: Branch; run bench
        run: |
          if [[ $PR_NUMBER ]]; then
            BASELINE_NAME=branch
            RESULTS_NAME=pr-$PR_NUMBER
            BENCH_FILTER='stdb_raw'
            echo "Running benchmarks without sqlite"
          else
            BASELINE_NAME=master
            RESULTS_NAME=$GITHUB_SHA
            BENCH_FILTER='(stdb_raw|sqlite)'
            echo "Running benchmarks with sqlite"
          fi
          pushd crates/bench
          rm -rf .spacetime
          cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
          # sticker price benchmark
          cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/.*/disk/update_bulk'
          cargo bench --bench special -- --save-baseline "$BASELINE_NAME"
          cargo run --bin summarize pack "$BASELINE_NAME"
          popd
          mkdir criterion-results
          [[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/
          cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json

      # this will work for both PR and master
      - name: Upload criterion results to DO spaces
        uses: shallwefootball/s3-upload-action@master
        with:
          aws_key_id: ${{ secrets.AWS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
          aws_bucket: "spacetimedb-ci-benchmarks"
          source_dir: criterion-results
          endpoint: https://nyc3.digitaloceanspaces.com
          destination_dir: benchmarks

      - name: Fetch markdown summary PR
        run: |
          if [[ $PR_NUMBER ]]; then
            OLD=master
            NEW=pr-$PR_NUMBER
          else
            OLD=$(git rev-parse HEAD~1)
            NEW=$GITHUB_SHA
          fi
          echo "fetching https://benchmarks.spacetimedb.com/compare/$OLD/$NEW"
          curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md

      - name: Post comment
        run: |
          BODY="<details><summary>Criterion benchmark results</summary>

          $(cat report.md)

          </details>"

          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"

      - name: Post failure comment
        if: ${{ failure() && env.COMMENT_UPDATE_URL }}
        run: |
          BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"

      - name: Clean up
        if: always()
        run: |
          rm -fr /stdb/*

  callgrind_benchmark:
    name: run callgrind benchmarks
    # DON'T run on benchmarks-runner, using docker on a self-hosted runner has
    # been broken for 4 years: https://github.com/actions/runner/issues/434 .
    # Fortunately, we can run on standard GitHub Actions infra because we don't care
    # about other stuff running on the machine!
    # runs-on: benchmarks-runner
    runs-on: ubuntu-latest
    timeout-minutes: 20 # on a successful run, runs in 8 minutes
    container:
      image: rust:1.93.0
      options: --privileged
    # filter for a comment containing 'benchmarks please'
    if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
    env:
      PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
    steps:
      - name: Clear stdb dir
        if: always()
        shell: bash
        run: |
          rm -fr /stdb/*

      - name: Install valgrind & iai-callgrind-runner
        run: |
          apt-get update
          apt-get install -y valgrind protobuf-compiler bash sudo curl gh
          cargo install --git https://github.com/clockworklabs/iai-callgrind.git --branch main iai-callgrind-runner
          git config --global --add safe.directory /__w/SpacetimeDB/SpacetimeDB

      # can't do this off self hosted:
      # - name: Enable CPU boost
      #   shell: bash
      #   run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost

      - name: Check membership
        if: ${{ github.event_name == 'issue_comment' }}
        env:
          CONTRIB_ORG: clockworklabs
          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
          ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
        shell: bash
        run: |
          curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
          if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
            gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
            exit 1
          fi

      - name: find PR branch
        if: ${{ env.PR_NUMBER }}
        shell: bash
        run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"

      - name: Checkout sources
        uses: actions/checkout@v3
        with:
          ref: ${{ env.PR_REF || github.ref }}
          # if we're on master we want to know what the sha of HEAD~1 is so
          # that we can compare results from it to HEAD (in the "Fetch markdown
          # summary PR" step). otherwise, we can use a fully shallow checkout
          fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}

      - name: Unbork GitHub Actions state
        shell: bash
        run: |
          echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
          echo "This is necessary because we are running as root inside a docker image"
          chmod -R a+rw .

      - name: Post initial comment
        shell: bash
        run: |
          set -exo pipefail
          if [[ $PR_NUMBER ]]; then
            comment_parent=issues/$PR_NUMBER
            comment_update=issues/comments
          else
            comment_parent=commits/$GITHUB_SHA
            comment_update=comments
          fi
          comment_body="Callgrind benchmark in progress..."
          comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
          echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV

      - name: Install stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          components: clippy
          toolchain: stable
          target: wasm32-unknown-unknown
          override: true

      - name: Build
        working-directory: crates/bench/
        shell: bash
        run: |
          cargo build --release

      - name: Install latest wasm-opt for module optimisations
        shell: bash
        run: |
          curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1

      # leave CPU boost on, doesn't affect callgrind!

      - name: Branch; run bench
        shell: bash
        run: |
          if [[ $PR_NUMBER ]]; then
            BASELINE_NAME=branch
            RESULTS_NAME=pr-$PR_NUMBER
            BENCH_FILTER='(special|stdb_module|stdb_raw)'
            echo "Running branch callgrind benchmarks"
          else
            BASELINE_NAME=master
            RESULTS_NAME=$GITHUB_SHA
            BENCH_FILTER='.*'
            echo "Running master callgrind benchmarks"
          fi
          pushd crates/bench
          rm -rf .spacetime
          cargo bench --bench callgrind -- --save-summary pretty-json
          cargo run --bin summarize pack-callgrind "$BASELINE_NAME"
          popd
          mkdir callgrind-results
          [[ ! $PR_NUMBER ]] && cp target/iai/$BASELINE_NAME.json callgrind-results/
          cp target/iai/$BASELINE_NAME.json callgrind-results/$RESULTS_NAME.json

      # this will work for both PR and master
      - name: Upload callgrind results to DO spaces
        uses: shallwefootball/s3-upload-action@master
        with:
          aws_key_id: ${{ secrets.AWS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
          aws_bucket: "spacetimedb-ci-benchmarks"
          source_dir: callgrind-results
          endpoint: https://nyc3.digitaloceanspaces.com
          destination_dir: callgrind-benchmarks

      - name: Fetch markdown summary PR
        shell: bash
        run: |
          if [[ $PR_NUMBER ]]; then
            OLD=master
            NEW=pr-$PR_NUMBER
          else
            OLD=$(git rev-parse HEAD~1)
            NEW=$GITHUB_SHA
          fi
          echo "fetching https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW"
          curl -sS https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW > report.md

      - name: Post comment
        shell: bash
        run: |
          BODY="<details><summary>Callgrind benchmark results</summary>
          $(cat report.md)
          </details>"
          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"

      - name: Post failure comment
        if: ${{ failure() && env.COMMENT_UPDATE_URL }}
        shell: bash
        run: |
          BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"

      - name: Clean up
        if: always()
        shell: bash
        run: |
          rm -fr /stdb/*
          echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
          echo "This is necessary because we are running as root inside a docker image"
          chmod -R a+rw .