ci: add workflow to build and cache the libpq

This avoid to attempt building the libpq concurrently with every Python
version, which will make caching fail.

diff --git a/.github/workflows/build-and-cache-libpq.yml b/.github/workflows/build-and-cache-libpq.yml
new file mode 100644 (file)
index 0000000..b6bae3c
--- /dev/null
+++ b/.github/workflows/build-and-cache-libpq.yml
@@ -0,0 +1,152 @@
+name: Build and cache libpq
+
+# Build the libpq package and cache the artifacts.
+#
+# Every Python version on the same architecture will use the same libpq.
+# Therefore building and caching the libpq together with the binary packages
+# result in multiple concurrent builds, and the github artifacts manageer very
+# confused.
+#
+# This job builds the libpq and then caches the artifacts so that the
+# packages-bin.yml workflow will find the library in the cache.
+#
+# You can see the caches at https://github.com/psycopg/psycopg/actions/caches
+#
+# Or from the API:
+#
+#   curl -fsSL -X GET \
+#     -H "Accept: application/vnd.github+json" \
+#     -H "Authorization: Bearer $GITHUB_TOKEN" \
+#     -H "X-GitHub-Api-Version: 2022-11-28" \
+#     "https://api.github.com/repos/psycopg/psycopg/actions/caches" \
+#   | jq -r '.actions_caches[].key'
+#
+# You can delete a cache using:
+#
+#   curl -fsSL -X DELETE \
+#     -H "Accept: application/vnd.github+json" \
+#     -H "Authorization: Bearer $GITHUB_TOKEN" \
+#     -H "X-GitHub-Api-Version: 2022-11-28" \
+#     "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0"
+#
+# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/build-and-cache-libpq.yml
+
+# TODO: move these env vars in an external env file in order to share them
+# across workflows.
+
+env:
+  # Latest release: https://www.postgresql.org/ftp/source/
+  LIBPQ_VERSION: "18.0"
+  # Note: On windows the latest version can be found at
+  # https://vcpkg.io/en/package/libpq
+  # However the command line tool doesn't have a flag to specify to install
+  # a specific version, so whatever you get you keep it.
+  # https://github.com/microsoft/vcpkg/discussions/25622
+
+  # Latest release: https://www.openssl.org/source/
+  OPENSSL_VERSION: "3.5.4"
+
+  # A string to differentiate build cacke keys to allow building different
+  # flavours of libpq in different branches without mixups. Currently used to
+  # make sure that libpq packages built for Psycopg 3.2 and 3.3 don't mix (in
+  # order to change the gssencmode default only on 3.3). We may use it for
+  # different reasons in the future.
+  PQ_FLAGS: "-gssencmode-disable"
+
+concurrency:
+  # Cancel older requests of the same workflow in the same branch.
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+
+  linux:  # {{{
+    runs-on: ubuntu-latest
+    if: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, ppc64le, aarch64]
+        platform: [manylinux, musllinux]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up QEMU for multi-arch build
+        # Check https://github.com/docker/setup-qemu-action for newer versions.
+        uses: docker/setup-qemu-action@v3
+        with:
+          # https://github.com/pypa/cibuildwheel/discussions/2256
+          image: tonistiigi/binfmt:qemu-v8.1.5
+
+      - name: Cache libpq build
+        uses: actions/cache@v4
+        with:
+          path: /tmp/libpq.build
+          key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.0
+        with:
+          package-dir: psycopg_c
+        env:
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
+          CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}}
+          CIBW_ARCHS_LINUX: auto aarch64 ppc64le
+          CIBW_BEFORE_ALL_LINUX: ./tools/ci/build_libpq.sh
+          CIBW_REPAIR_WHEEL_COMMAND: >-
+            ./tools/ci/strip_wheel.sh {wheel}
+            && auditwheel repair -w {dest_dir} {wheel}
+          CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION
+          CIBW_ENVIRONMENT: >-
+            LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+            LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64"
+
+
+  # }}}
+
+  macos:  # {{{
+    runs-on: macos-latest
+    if: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, arm64]
+
+    steps:
+      - name: Checkout repos
+        uses: actions/checkout@v5
+
+      - name: Cache libpq build
+        uses: actions/cache@v4
+        with:
+          path: /tmp/libpq.build
+          key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.0
+        with:
+          package-dir: psycopg_c
+        env:
+          CIBW_BUILD: cp313-macosx_${{matrix.arch}}
+          CIBW_ARCHS_MACOS: ${{matrix.arch}}
+          MACOSX_ARCHITECTURE: ${{matrix.arch}}
+          CIBW_BEFORE_ALL_MACOS: ./tools/ci/build_libpq.sh
+          CIBW_ENVIRONMENT: >-
+            PSYCOPG_IMPL=binary
+            LIBPQ_BUILD_PREFIX=/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+
+  # }}}

diff --git a/.github/workflows/packages-bin.yml b/.github/workflows/packages-bin.yml
index df41e3fcabad51eff537352fe3c158d8ca8394c7..a48320c5e988db4a1df440920129696120d9c69d 100644 (file)
--- a/.github/workflows/packages-bin.yml
+++ b/.github/workflows/packages-bin.yml
@@ -1,28 +1,11 @@
 name: Build binary packages
 
-# Note: Libpq is currently built from source on most platforms and the build
-# artifacts are cached across pipeline runs.
+# Note: these jobs also build and cache the libpq, but, because every Python
+# version will try to build and cache the same libpq instance, there is a race
+# condition and most likely the artifacts manager will refuse to cache.
 #
-# You can see the caches at https://github.com/psycopg/psycopg/actions/caches
-#
-# Or from the API:
-#
-#   curl -fsSL -X GET \
-#     -H "Accept: application/vnd.github+json" \
-#     -H "Authorization: Bearer $GITHUB_TOKEN" \
-#     -H "X-GitHub-Api-Version: 2022-11-28" \
-#     "https://api.github.com/repos/psycopg/psycopg/actions/caches" \
-#   | jq -r '.actions_caches[].key'
-#
-# You can delete a cache using:
-#
-#   curl -fsSL -X DELETE \
-#     -H "Accept: application/vnd.github+json" \
-#     -H "Authorization: Bearer $GITHUB_TOKEN" \
-#     -H "X-GitHub-Api-Version: 2022-11-28" \
-#     "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0"
-#
-# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key
+# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl
+# version change in order to update the cache.
 
 on:
   push: