gh-142466: Pixi package definitions for downstream use (#142469)

* WIP: ENH: Pixi package definitions for downstream development

[skip ci]

* linux-64 support

* tidy gitignore

* respond to review
- switch cases on `PYTHON_VARIANT`
- remove `minor_version` by using `python3`
- remove runtime-only asan options

* README updates

* use `.md` to preview rendering

* Apply suggestions from code review

Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
* Apply suggestion from @FFY00

Co-authored-by: Filipe Laíns <filipe.lains@gmail.com>
* Apply suggestion from @FFY00

Co-authored-by: Filipe Laíns <filipe.lains@gmail.com>
* Apply suggestion from @FFY00

Co-authored-by: Filipe Laíns <filipe.lains@gmail.com>
* Apply suggestion from @lucascolley

---------

Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Co-authored-by: Filipe Laíns <filipe.lains@gmail.com>

diff --git a/.gitignore b/.gitignore
index 4ea2fd9655471d867d15c633725a6e8fa386e27a..e234d86e8d5532705625beaa13c33add2481cb68 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -45,6 +45,7 @@ gmon.out
 .pytest_cache/
 .ruff_cache/
 .DS_Store
+.pixi/
 
 *.exe
 

diff --git a/Tools/README b/Tools/README
index 22d76dfdbcf4a4bccbc494feb6cdbaaaeaaee6fd..90acb2614820ee45c4c140fee2fdad25e91aab1b 100644 (file)
--- a/Tools/README
+++ b/Tools/README
@@ -43,6 +43,8 @@ patchcheck      Tools for checking and applying patches to the Python source cod
 
 peg_generator   PEG-based parser generator (pegen) used for new parser.
 
+pixi-packages   Pixi package definitions for downstream from-source builds.
+
 scripts         A number of useful single-file programs, e.g. run_tests.py
                 which runs the Python test suite.
 

diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md
new file mode 100644 (file)
index 0000000..50c3315
--- /dev/null
+++ b/Tools/pixi-packages/README.md
@@ -0,0 +1,36 @@
+# CPython Pixi packages
+
+This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section)
+which can be built from the CPython source code.
+
+Downstream developers can make use of these packages by adding them as Git dependencies in a
+[Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
+
+```toml
+[dependencies]
+python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi-packages/asan" }
+```
+
+This is particularly useful when developers need to build CPython from source
+(for example, for an ASan-instrumented build), as it does not require any manual
+clone or build steps. Instead, Pixi will automatically handle both the build
+and installation of the package.
+
+Each package definition is contained in a subdirectory, but they share the build script
+`build.sh` in this directory. Currently defined package variants:
+
+- `default`
+- `asan`: ASan-instrumented build with `PYTHON_ASAN=1`
+
+## Maintenance
+
+- Keep the `version` fields in each `recipe.yaml` up to date with the Python version
+- Keep the dependency requirements up to date in each `recipe.yaml`
+- Update `build.sh` for any breaking changes in the `configure` and `make` workflow
+
+## Opportunities for future improvement
+
+- More package variants (such as TSan, UBSan)
+- Support for Windows
+- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599
+- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved

diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml
new file mode 100644 (file)
index 0000000..001ff78
--- /dev/null
+++ b/Tools/pixi-packages/asan/pixi.toml
@@ -0,0 +1,8 @@
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml
new file mode 100644 (file)
index 0000000..dea8839
--- /dev/null
+++ b/Tools/pixi-packages/asan/recipe.yaml
@@ -0,0 +1,63 @@
+context:
+  # Keep up to date
+  version: "3.15"
+
+package:
+  name: python
+  version: ${{ version }}
+
+source:
+  - path: ../../..
+
+build:
+  files:
+    exclude:
+      - "*.o"
+  script:
+    file: ../build.sh
+    env:
+      PYTHON_VARIANT: "asan"
+
+# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+requirements:
+  build:
+    - ${{ compiler('c') }}
+    - ${{ compiler('cxx') }}
+    - make
+    - pkg-config
+    # configure script looks for llvm-ar for lto
+    - if: osx
+      then:
+        - llvm-tools
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - binutils_impl_${{ target_platform }}
+    - clang-19
+    - llvm-tools-19
+
+  host:
+    - bzip2
+    - sqlite
+    - liblzma-devel
+    - zlib
+    - zstd
+    - openssl
+    - readline
+    - tk
+    # These two are just to get the headers needed for tk.h, but is unused
+    - xorg-libx11
+    - xorg-xorgproto
+    - ncurses
+    - libffi
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - libuuid
+    - libmpdec-devel
+    - expat
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE

diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh
new file mode 100644 (file)
index 0000000..120f1d6
--- /dev/null
+++ b/Tools/pixi-packages/build.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+if [[ "${PYTHON_VARIANT}" == "asan" ]]; then
+    echo "BUILD TYPE: ASAN"
+    BUILD_DIR="../build_asan"
+    CONFIGURE_EXTRA="--with-address-sanitizer"
+    export PYTHON_ASAN="1"
+    export ASAN_OPTIONS="strict_init_order=true"
+else
+    echo "BUILD TYPE: DEFAULT"
+    BUILD_DIR="../build"
+    CONFIGURE_EXTRA=""
+fi
+
+mkdir -p "${BUILD_DIR}"
+cd "${BUILD_DIR}"
+
+if [[ -f configure-done ]]; then
+    echo "Skipping configure step, already done."
+else
+    "${SRC_DIR}/configure" \
+        --prefix="${PREFIX}" \
+        --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \
+        --enable-shared \
+        --srcdir="${SRC_DIR}" \
+        ${CONFIGURE_EXTRA}
+fi
+
+touch configure-done
+
+make -j"${CPU_COUNT}" install
+ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python"
+
+# https://github.com/prefix-dev/rattler-build/issues/2012
+if [[ ${OSTYPE} == "darwin"* ]]; then
+    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib"
+fi

diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml
new file mode 100644 (file)
index 0000000..001ff78
--- /dev/null
+++ b/Tools/pixi-packages/default/pixi.toml
@@ -0,0 +1,8 @@
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/default/recipe.yaml
new file mode 100644 (file)
index 0000000..eeb4052
--- /dev/null
+++ b/Tools/pixi-packages/default/recipe.yaml
@@ -0,0 +1,61 @@
+context:
+  # Keep up to date
+  version: "3.15"
+
+package:
+  name: python
+  version: ${{ version }}
+
+source:
+  - path: ../../..
+
+build:
+  files:
+    exclude:
+      - "*.o"
+  script:
+    file: ../build.sh
+
+# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+requirements:
+  build:
+    - ${{ compiler('c') }}
+    - ${{ compiler('cxx') }}
+    - make
+    - pkg-config
+    # configure script looks for llvm-ar for lto
+    - if: osx
+      then:
+        - llvm-tools
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - binutils_impl_${{ target_platform }}
+    - clang-19
+    - llvm-tools-19
+
+  host:
+    - bzip2
+    - sqlite
+    - liblzma-devel
+    - zlib
+    - zstd
+    - openssl
+    - readline
+    - tk
+    # These two are just to get the headers needed for tk.h, but is unused
+    - xorg-libx11
+    - xorg-xorgproto
+    - ncurses
+    - libffi
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - libuuid
+    - libmpdec-devel
+    - expat
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE