astral-ruff/.github/workflows/typing_conformance.yaml

name: Run typing conformance

permissions: {}

on:
  pull_request:
    paths:
      - "crates/ty*/**"
      - "!crates/ty_ide/**"
      - "!crates/ty_server/**"
      - "!crates/ty_test/**"
      - "!crates/ty_completion_eval/**"
      - "!crates/ty_wasm/**"
      - "crates/ruff_db"
      - "crates/ruff_python_ast"
      - "crates/ruff_python_parser"
      - "scripts/conformance.py"
      - ".github/workflows/typing_conformance.yaml"
      - "Cargo.lock"
      - "!**.md"
      - "!**.snap"
      # It's tempting to skip all Python files in every directory, but changes to
      # Python files in `ty_vendored` can affect the output of the ecosystem analysis,
      # so we apply a narrow exemption for all files in the corpus directory instead.
      - "!crates/ty_python_semantic/resources/corpus/**"

concurrency:
  group: typing-conformance-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CARGO_TERM_COLOR: always
  RUSTUP_MAX_RETRIES: 10
  RUST_BACKTRACE: 1
  # Line-tables-only debug info: faster builds, backtraces still work.
  CARGO_PROFILE_DEV_DEBUG: line-tables-only
  CONFORMANCE_SUITE_COMMIT: 5a701a037b5243df1f39622b642893d865f06205
  PYTHON_VERSION: 3.12

jobs:
  typing_conformance:
    name: Compute diagnostic diff
    runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-32' || 'ubuntu-latest' }}
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: ruff
          fetch-depth: 0
          persist-credentials: false

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: python/typing
          ref: ${{ env.CONFORMANCE_SUITE_COMMIT }}
          path: typing
          persist-credentials: false

      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
        with:
          workspaces: "ruff"

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Rust toolchain
        run: rustup show

      - name: "Install mold"
        uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1

      - name: Compute diagnostic diff
        shell: bash
        run: |
          # Build the executable for the old and new commit
          (
            cd ruff

            echo "new commit"
            git rev-list --format=%s --max-count=1 "$GITHUB_SHA"
            cargo build --bin ty
            mv target/debug/ty ty-new

            MERGE_BASE="$(git merge-base "$GITHUB_SHA" "origin/$GITHUB_BASE_REF")"
            git checkout -b old_commit "$MERGE_BASE"
            echo "old commit (merge base)"
            git rev-list --format=%s --max-count=1 old_commit
            cargo build --bin ty
            mv target/debug/ty ty-old
          )

          (
            echo "Creating comment with conformance comparison"
            cd ruff
            git switch - --detach

            python "./scripts/conformance.py" \
             --old-ty "./ty-old" \
             --new-ty "./ty-new" \
             --tests-path "${GITHUB_WORKSPACE}/typing/conformance/" \
             --python-version "$PYTHON_VERSION" \
             --output ../typing_conformance_diagnostics.diff
          )

      # NOTE: astral-sh-bot uses this artifact to post comments on PRs.
      # Make sure to update the bot if you rename the artifact.
      - name: Upload diff
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: typing_conformance_diagnostics_diff
          path: typing_conformance_diagnostics.diff