From 8bb5b6e8ce61da41b5affd5eb12d9cc46b5af448 Mon Sep 17 00:00:00 2001 From: Lucas Colley Date: Mon, 15 Dec 2025 15:04:16 +0000 Subject: [PATCH] gh-142466: Pixi package definitions for downstream use (#142469) MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * WIP: ENH: Pixi package definitions for downstream development [skip ci] * linux-64 support * tidy gitignore * respond to review - switch cases on `PYTHON_VARIANT` - remove `minor_version` by using `python3` - remove runtime-only asan options * README updates * use `.md` to preview rendering * Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> * Apply suggestion from @FFY00 Co-authored-by: Filipe Laíns * Apply suggestion from @FFY00 Co-authored-by: Filipe Laíns * Apply suggestion from @FFY00 Co-authored-by: Filipe Laíns * Apply suggestion from @lucascolley --------- Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Filipe Laíns --- .gitignore | 1 + Tools/README | 2 + Tools/pixi-packages/README.md | 36 ++++++++++++++ Tools/pixi-packages/asan/pixi.toml | 8 ++++ Tools/pixi-packages/asan/recipe.yaml | 63 +++++++++++++++++++++++++ Tools/pixi-packages/build.sh | 37 +++++++++++++++ Tools/pixi-packages/default/pixi.toml | 8 ++++ Tools/pixi-packages/default/recipe.yaml | 61 ++++++++++++++++++++++++ 8 files changed, 216 insertions(+) create mode 100644 Tools/pixi-packages/README.md create mode 100644 Tools/pixi-packages/asan/pixi.toml create mode 100644 Tools/pixi-packages/asan/recipe.yaml create mode 100644 Tools/pixi-packages/build.sh create mode 100644 Tools/pixi-packages/default/pixi.toml create mode 100644 Tools/pixi-packages/default/recipe.yaml diff --git a/.gitignore b/.gitignore index 4ea2fd965547..e234d86e8d55 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ gmon.out .pytest_cache/ .ruff_cache/ .DS_Store +.pixi/ *.exe diff --git a/Tools/README b/Tools/README index 22d76dfdbcf4..90acb2614820 100644 --- a/Tools/README +++ b/Tools/README @@ -43,6 +43,8 @@ patchcheck Tools for checking and applying patches to the Python source cod peg_generator PEG-based parser generator (pegen) used for new parser. +pixi-packages Pixi package definitions for downstream from-source builds. + scripts A number of useful single-file programs, e.g. run_tests.py which runs the Python test suite. diff --git a/Tools/pixi-packages/README.md b/Tools/pixi-packages/README.md new file mode 100644 index 000000000000..50c3315ac0e5 --- /dev/null +++ b/Tools/pixi-packages/README.md @@ -0,0 +1,36 @@ +# CPython Pixi packages + +This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) +which can be built from the CPython source code. + +Downstream developers can make use of these packages by adding them as Git dependencies in a +[Pixi workspace](https://pixi.sh/latest/first_workspace/), like: + +```toml +[dependencies] +python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi-packages/asan" } +``` + +This is particularly useful when developers need to build CPython from source +(for example, for an ASan-instrumented build), as it does not require any manual +clone or build steps. Instead, Pixi will automatically handle both the build +and installation of the package. + +Each package definition is contained in a subdirectory, but they share the build script +`build.sh` in this directory. Currently defined package variants: + +- `default` +- `asan`: ASan-instrumented build with `PYTHON_ASAN=1` + +## Maintenance + +- Keep the `version` fields in each `recipe.yaml` up to date with the Python version +- Keep the dependency requirements up to date in each `recipe.yaml` +- Update `build.sh` for any breaking changes in the `configure` and `make` workflow + +## Opportunities for future improvement + +- More package variants (such as TSan, UBSan) +- Support for Windows +- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599 +- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved diff --git a/Tools/pixi-packages/asan/pixi.toml b/Tools/pixi-packages/asan/pixi.toml new file mode 100644 index 000000000000..001ff78fa5d8 --- /dev/null +++ b/Tools/pixi-packages/asan/pixi.toml @@ -0,0 +1,8 @@ +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["osx-arm64", "linux-64"] +preview = ["pixi-build"] + +[package.build.backend] +name = "pixi-build-rattler-build" +version = "*" diff --git a/Tools/pixi-packages/asan/recipe.yaml b/Tools/pixi-packages/asan/recipe.yaml new file mode 100644 index 000000000000..dea88394ad9f --- /dev/null +++ b/Tools/pixi-packages/asan/recipe.yaml @@ -0,0 +1,63 @@ +context: + # Keep up to date + version: "3.15" + +package: + name: python + version: ${{ version }} + +source: + - path: ../../.. + +build: + files: + exclude: + - "*.o" + script: + file: ../build.sh + env: + PYTHON_VARIANT: "asan" + +# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml +requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + - if: linux + then: + - ld_impl_${{ target_platform }} + - binutils_impl_${{ target_platform }} + - clang-19 + - llvm-tools-19 + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - ld_impl_${{ target_platform }} + - libuuid + - libmpdec-devel + - expat + +about: + homepage: https://www.python.org/ + license: Python-2.0 + license_file: LICENSE diff --git a/Tools/pixi-packages/build.sh b/Tools/pixi-packages/build.sh new file mode 100644 index 000000000000..120f1d6bb008 --- /dev/null +++ b/Tools/pixi-packages/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [[ "${PYTHON_VARIANT}" == "asan" ]]; then + echo "BUILD TYPE: ASAN" + BUILD_DIR="../build_asan" + CONFIGURE_EXTRA="--with-address-sanitizer" + export PYTHON_ASAN="1" + export ASAN_OPTIONS="strict_init_order=true" +else + echo "BUILD TYPE: DEFAULT" + BUILD_DIR="../build" + CONFIGURE_EXTRA="" +fi + +mkdir -p "${BUILD_DIR}" +cd "${BUILD_DIR}" + +if [[ -f configure-done ]]; then + echo "Skipping configure step, already done." +else + "${SRC_DIR}/configure" \ + --prefix="${PREFIX}" \ + --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \ + --enable-shared \ + --srcdir="${SRC_DIR}" \ + ${CONFIGURE_EXTRA} +fi + +touch configure-done + +make -j"${CPU_COUNT}" install +ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python" + +# https://github.com/prefix-dev/rattler-build/issues/2012 +if [[ ${OSTYPE} == "darwin"* ]]; then + cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib" +fi diff --git a/Tools/pixi-packages/default/pixi.toml b/Tools/pixi-packages/default/pixi.toml new file mode 100644 index 000000000000..001ff78fa5d8 --- /dev/null +++ b/Tools/pixi-packages/default/pixi.toml @@ -0,0 +1,8 @@ +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["osx-arm64", "linux-64"] +preview = ["pixi-build"] + +[package.build.backend] +name = "pixi-build-rattler-build" +version = "*" diff --git a/Tools/pixi-packages/default/recipe.yaml b/Tools/pixi-packages/default/recipe.yaml new file mode 100644 index 000000000000..eeb4052ec385 --- /dev/null +++ b/Tools/pixi-packages/default/recipe.yaml @@ -0,0 +1,61 @@ +context: + # Keep up to date + version: "3.15" + +package: + name: python + version: ${{ version }} + +source: + - path: ../../.. + +build: + files: + exclude: + - "*.o" + script: + file: ../build.sh + +# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml +requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - make + - pkg-config + # configure script looks for llvm-ar for lto + - if: osx + then: + - llvm-tools + - if: linux + then: + - ld_impl_${{ target_platform }} + - binutils_impl_${{ target_platform }} + - clang-19 + - llvm-tools-19 + + host: + - bzip2 + - sqlite + - liblzma-devel + - zlib + - zstd + - openssl + - readline + - tk + # These two are just to get the headers needed for tk.h, but is unused + - xorg-libx11 + - xorg-xorgproto + - ncurses + - libffi + - if: linux + then: + - ld_impl_${{ target_platform }} + - libuuid + - libmpdec-devel + - expat + +about: + homepage: https://www.python.org/ + license: Python-2.0 + license_file: LICENSE -- 2.47.3