From e11da8ae4013974614bb9acde72fb603eb4e96e5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Oct 2025 02:28:06 +0200 Subject: [PATCH] ci: add workflow to build and cache the libpq This avoid to attempt building the libpq concurrently with every Python version, which will make caching fail. --- .github/workflows/build-and-cache-libpq.yml | 152 ++++++++++++++++++++ .github/workflows/packages-bin.yml | 27 +--- 2 files changed, 157 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/build-and-cache-libpq.yml diff --git a/.github/workflows/build-and-cache-libpq.yml b/.github/workflows/build-and-cache-libpq.yml new file mode 100644 index 000000000..b6bae3c1e --- /dev/null +++ b/.github/workflows/build-and-cache-libpq.yml @@ -0,0 +1,152 @@ +name: Build and cache libpq + +# Build the libpq package and cache the artifacts. +# +# Every Python version on the same architecture will use the same libpq. +# Therefore building and caching the libpq together with the binary packages +# result in multiple concurrent builds, and the github artifacts manageer very +# confused. +# +# This job builds the libpq and then caches the artifacts so that the +# packages-bin.yml workflow will find the library in the cache. +# +# You can see the caches at https://github.com/psycopg/psycopg/actions/caches +# +# Or from the API: +# +# curl -fsSL -X GET \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches" \ +# | jq -r '.actions_caches[].key' +# +# You can delete a cache using: +# +# curl -fsSL -X DELETE \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0" +# +# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/build-and-cache-libpq.yml + +# TODO: move these env vars in an external env file in order to share them +# across workflows. + +env: + # Latest release: https://www.postgresql.org/ftp/source/ + LIBPQ_VERSION: "18.0" + # Note: On windows the latest version can be found at + # https://vcpkg.io/en/package/libpq + # However the command line tool doesn't have a flag to specify to install + # a specific version, so whatever you get you keep it. + # https://github.com/microsoft/vcpkg/discussions/25622 + + # Latest release: https://www.openssl.org/source/ + OPENSSL_VERSION: "3.5.4" + + # A string to differentiate build cacke keys to allow building different + # flavours of libpq in different branches without mixups. Currently used to + # make sure that libpq packages built for Psycopg 3.2 and 3.3 don't mix (in + # order to change the gssencmode default only on 3.3). We may use it for + # different reasons in the future. + PQ_FLAGS: "-gssencmode-disable" + +concurrency: + # Cancel older requests of the same workflow in the same branch. + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + + linux: # {{{ + runs-on: ubuntu-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, ppc64le, aarch64] + platform: [manylinux, musllinux] + + steps: + - uses: actions/checkout@v5 + + - name: Set up QEMU for multi-arch build + # Check https://github.com/docker/setup-qemu-action for newer versions. + uses: docker/setup-qemu-action@v3 + with: + # https://github.com/pypa/cibuildwheel/discussions/2256 + image: tonistiigi/binfmt:qemu-v8.1.5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + with: + package-dir: psycopg_c + env: + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2014 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}} + CIBW_ARCHS_LINUX: auto aarch64 ppc64le + CIBW_BEFORE_ALL_LINUX: ./tools/ci/build_libpq.sh + CIBW_REPAIR_WHEEL_COMMAND: >- + ./tools/ci/strip_wheel.sh {wheel} + && auditwheel repair -w {dest_dir} {wheel} + CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION + CIBW_ENVIRONMENT: >- + LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64" + + + # }}} + + macos: # {{{ + runs-on: macos-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, arm64] + + steps: + - name: Checkout repos + uses: actions/checkout@v5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + with: + package-dir: psycopg_c + env: + CIBW_BUILD: cp313-macosx_${{matrix.arch}} + CIBW_ARCHS_MACOS: ${{matrix.arch}} + MACOSX_ARCHITECTURE: ${{matrix.arch}} + CIBW_BEFORE_ALL_MACOS: ./tools/ci/build_libpq.sh + CIBW_ENVIRONMENT: >- + PSYCOPG_IMPL=binary + LIBPQ_BUILD_PREFIX=/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + + # }}} diff --git a/.github/workflows/packages-bin.yml b/.github/workflows/packages-bin.yml index df41e3fca..a48320c5e 100644 --- a/.github/workflows/packages-bin.yml +++ b/.github/workflows/packages-bin.yml @@ -1,28 +1,11 @@ name: Build binary packages -# Note: Libpq is currently built from source on most platforms and the build -# artifacts are cached across pipeline runs. +# Note: these jobs also build and cache the libpq, but, because every Python +# version will try to build and cache the same libpq instance, there is a race +# condition and most likely the artifacts manager will refuse to cache. # -# You can see the caches at https://github.com/psycopg/psycopg/actions/caches -# -# Or from the API: -# -# curl -fsSL -X GET \ -# -H "Accept: application/vnd.github+json" \ -# -H "Authorization: Bearer $GITHUB_TOKEN" \ -# -H "X-GitHub-Api-Version: 2022-11-28" \ -# "https://api.github.com/repos/psycopg/psycopg/actions/caches" \ -# | jq -r '.actions_caches[].key' -# -# You can delete a cache using: -# -# curl -fsSL -X DELETE \ -# -H "Accept: application/vnd.github+json" \ -# -H "Authorization: Bearer $GITHUB_TOKEN" \ -# -H "X-GitHub-Api-Version: 2022-11-28" \ -# "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0" -# -# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key +# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl +# version change in order to update the cache. on: push: -- 2.47.3