gh-143120: pixi builds for free-threading and TSAN (#142872)

Co-authored-by: Isuru Fernando <isuruf@gmail.com>
Co-authored-by: Lucas Colley <lucas.colley8@gmail.com>

Tools/pixi-packages/README.md

@@ -1,18 +1,20 @@
 # CPython Pixi packages
 
-This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section)
-which can be built from the CPython source code.
+This directory contains definitions for [Pixi
+packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can
+be built from the CPython source code.
 
-Downstream developers can make use of these packages by adding them as Git dependencies in a
-[Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
+Downstream developers can make use of these packages by adding them as Git dependencies
+in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
 
 ```toml
 [dependencies]
-python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi-packages/asan" }
+python.git = "https://github.com/python/cpython"
+python.subdirectory = "Tools/pixi-packages/asan"
 ```
 
 This is particularly useful when developers need to build CPython from source
-(for example, for an ASan-instrumented build), as it does not require any manual
+(for example, for an ASan or TSan-instrumented build), as it does not require any manual
 clone or build steps. Instead, Pixi will automatically handle both the build
 and installation of the package.
 
@@ -20,17 +22,35 @@ Each package definition is contained in a subdirectory, but they share the build
 `build.sh` in this directory. Currently defined package variants:
 
 - `default`
-- `asan`: ASan-instrumented build with `PYTHON_ASAN=1`
+- `freethreading`
+- `asan`: ASan-instrumented build
+- `tsan-freethreading`: TSan-instrumented free-threading build
 
 ## Maintenance
 
-- Keep the `version` fields in each `recipe.yaml` up to date with the Python version
-- Keep the dependency requirements up to date in each `recipe.yaml`
+- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the
+  Python version
 - Update `build.sh` for any breaking changes in the `configure` and `make` workflow
 
 ## Opportunities for future improvement
 
-- More package variants (such as TSan, UBSan)
+- More package variants (such as UBSan)
 - Support for Windows
-- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599
-- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved
+- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on
+  [pixi-build-backends#532](https://github.com/prefix-dev/pixi-build-backends/pull/532)
+  and [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248)
+
+## Troubleshooting
+
+TSan builds may crash on Linux with
+```
+FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000
+```
+To fix it, try reducing `mmap_rnd_bits`:
+
+```bash
+$ sudo sysctl vm.mmap_rnd_bits
+vm.mmap_rnd_bits = 32  # too high for TSan
+$ sudo sysctl vm.mmap_rnd_bits=28  # reduce it
+vm.mmap_rnd_bits = 28
+```

Tools/pixi-packages/asan/pixi.toml

@@ -1,6 +1,9 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 [workspace]
 channels = ["https://prefix.dev/conda-forge"]
-platforms = ["osx-arm64", "linux-64"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
 preview = ["pixi-build"]
 
 [package.build.backend]

Tools/pixi-packages/asan/recipe.yaml

@@ -1,61 +1,92 @@
+# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 context:
   # Keep up to date
-  version: "3.15"
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
 
-package:
+recipe:
   name: python
-  version: ${{ version }}
 
 source:
   - path: ../../..
 
-build:
-  files:
-    exclude:
-      - "*.o"
-  script:
-    file: ../build.sh
-    env:
-      PYTHON_VARIANT: "asan"
-
-# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
-requirements:
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
   build:
-    - ${{ compiler('c') }}
-    - ${{ compiler('cxx') }}
-    - make
-    - pkg-config
-    # configure script looks for llvm-ar for lto
-    - if: osx
-      then:
-        - llvm-tools
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - binutils_impl_${{ target_platform }}
-    - clang-19
-    - llvm-tools-19
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
 
-  host:
-    - bzip2
-    - sqlite
-    - liblzma-devel
-    - zlib
-    - zstd
-    - openssl
-    - readline
-    - tk
-    # These two are just to get the headers needed for tk.h, but is unused
-    - xorg-libx11
-    - xorg-xorgproto
-    - ncurses
-    - libffi
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - libuuid
-    - libmpdec-devel
-    - expat
+- package:
+    name: python
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      # Note that we are not using stdlib arguments which means the packages
+      # are built for the build settings and are not relocatable to a different
+      # machine that has a older system version. (eg: macOS/glibc version)
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: linux and "san" in variant
+        then:
+          - libsanitizer
+      - if: osx and "san" in variant
+        then:
+          - libcompiler-rt
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
 
 about:
   homepage: https://www.python.org/

Tools/pixi-packages/asan/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - asan
+abi_tag:
+  - asan_cp315
+version:
+  - 3.15

Tools/pixi-packages/build.sh

@@ -1,17 +1,36 @@
 #!/bin/bash
 
-if [[ "${PYTHON_VARIANT}" == "asan" ]]; then
-    echo "BUILD TYPE: ASAN"
-    BUILD_DIR="../build_asan"
+echo "PYTHON_VARIANT: ${PYTHON_VARIANT}"
+
+if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil"
+elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then
     CONFIGURE_EXTRA="--with-address-sanitizer"
-    export PYTHON_ASAN="1"
     export ASAN_OPTIONS="strict_init_order=true"
-else
-    echo "BUILD TYPE: DEFAULT"
-    BUILD_DIR="../build"
+elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer"
+    export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt"
+elif [[ "${PYTHON_VARIANT}" == "default" ]]; then
     CONFIGURE_EXTRA=""
+else
+    echo "Unknown PYTHON_VARIANT: ${PYTHON_VARIANT}"
+    exit 1
 fi
 
+# rattler-build by default set a target of 10.9
+# override it to at least 10.12
+case ${MACOSX_DEPLOYMENT_TARGET:-10.12} in
+    10.12|10.13|10.14|10.15|10.16)
+        ;;
+    10.*)
+        export CPPFLAGS=${CPPFLAGS/-mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}/-mmacosx-version-min=10.12}
+        export MACOSX_DEPLOYMENT_TARGET=10.12
+        ;;
+    *)
+        ;;
+esac
+
+BUILD_DIR="../build_${PYTHON_VARIANT}"
 mkdir -p "${BUILD_DIR}"
 cd "${BUILD_DIR}"
 
@@ -23,6 +42,7 @@ else
         --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \
         --enable-shared \
         --srcdir="${SRC_DIR}" \
+        --with-system-expat \
         ${CONFIGURE_EXTRA}
 fi
 
@@ -30,8 +50,3 @@ touch configure-done
 
 make -j"${CPU_COUNT}" install
 ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python"
-
-# https://github.com/prefix-dev/rattler-build/issues/2012
-if [[ ${OSTYPE} == "darwin"* ]]; then
-    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib"
-fi

Tools/pixi-packages/clone-recipe.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Please always only modify default/recipe.yaml and default/pixi.toml and then run this
+# script to propagate the changes to the other variants.
+set -o errexit
+cd "$(dirname "$0")"
+
+for variant in asan freethreading tsan-freethreading; do
+    cp -av default/recipe.yaml default/pixi.toml ${variant}/
+done

Tools/pixi-packages/default/pixi.toml

@@ -1,6 +1,9 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 [workspace]
 channels = ["https://prefix.dev/conda-forge"]
-platforms = ["osx-arm64", "linux-64"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
 preview = ["pixi-build"]
 
 [package.build.backend]

Tools/pixi-packages/default/recipe.yaml

@@ -1,59 +1,92 @@
+# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 context:
   # Keep up to date
-  version: "3.15"
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
 
-package:
+recipe:
   name: python
-  version: ${{ version }}
 
 source:
   - path: ../../..
 
-build:
-  files:
-    exclude:
-      - "*.o"
-  script:
-    file: ../build.sh
-
-# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
-requirements:
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
   build:
-    - ${{ compiler('c') }}
-    - ${{ compiler('cxx') }}
-    - make
-    - pkg-config
-    # configure script looks for llvm-ar for lto
-    - if: osx
-      then:
-        - llvm-tools
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - binutils_impl_${{ target_platform }}
-    - clang-19
-    - llvm-tools-19
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
 
-  host:
-    - bzip2
-    - sqlite
-    - liblzma-devel
-    - zlib
-    - zstd
-    - openssl
-    - readline
-    - tk
-    # These two are just to get the headers needed for tk.h, but is unused
-    - xorg-libx11
-    - xorg-xorgproto
-    - ncurses
-    - libffi
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - libuuid
-    - libmpdec-devel
-    - expat
+- package:
+    name: python
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      # Note that we are not using stdlib arguments which means the packages
+      # are built for the build settings and are not relocatable to a different
+      # machine that has a older system version. (eg: macOS/glibc version)
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: linux and "san" in variant
+        then:
+          - libsanitizer
+      - if: osx and "san" in variant
+        then:
+          - libcompiler-rt
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
 
 about:
   homepage: https://www.python.org/

Tools/pixi-packages/default/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - default
+abi_tag:
+  - cp315
+version:
+  - 3.15

Tools/pixi-packages/freethreading/pixi.toml

@@ -0,0 +1,11 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

Tools/pixi-packages/freethreading/recipe.yaml

@@ -0,0 +1,94 @@
+# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
+context:
+  # Keep up to date
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
+
+recipe:
+  name: python
+
+source:
+  - path: ../../..
+
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
+
+- package:
+    name: python
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      # Note that we are not using stdlib arguments which means the packages
+      # are built for the build settings and are not relocatable to a different
+      # machine that has a older system version. (eg: macOS/glibc version)
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: linux and "san" in variant
+        then:
+          - libsanitizer
+      - if: osx and "san" in variant
+        then:
+          - libcompiler-rt
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE

Tools/pixi-packages/freethreading/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - freethreading
+abi_tag:
+  - cp315t
+version:
+  - 3.15

Tools/pixi-packages/tsan-freethreading/pixi.toml

@@ -0,0 +1,11 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

Tools/pixi-packages/tsan-freethreading/recipe.yaml

@@ -0,0 +1,94 @@
+# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
+context:
+  # Keep up to date
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
+
+recipe:
+  name: python
+
+source:
+  - path: ../../..
+
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
+
+- package:
+    name: python
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      # Note that we are not using stdlib arguments which means the packages
+      # are built for the build settings and are not relocatable to a different
+      # machine that has a older system version. (eg: macOS/glibc version)
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: linux and "san" in variant
+        then:
+          - libsanitizer
+      - if: osx and "san" in variant
+        then:
+          - libcompiler-rt
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE

Tools/pixi-packages/tsan-freethreading/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - tsan-freethreading
+abi_tag:
+  - tsan_cp315t
+version:
+  - 3.15