From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sun, 5 Jan 2025 23:48:36 +0000 (+0100)
Subject: ci(macos): build libpq instead of using Homebrew libraries
X-Git-Tag: 3.2.4~4^2~2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bdba7673873bb5c8abb5edfd474cc16cf705b879;p=thirdparty%2Fpsycopg.git

ci(macos): build libpq instead of using Homebrew libraries

This avoids problems with MACOSX_DEPLOYMENT_TARGET drifting in builders
(see #858).

Largely contributed by @bwoodsend in https://github.com/psycopg/psycopg2/pull/1755.
---

diff --git a/.github/workflows/packages-bin.yml b/.github/workflows/packages-bin.yml
index 6102be107..6bf7f1ffb 100644
--- a/.github/workflows/packages-bin.yml
+++ b/.github/workflows/packages-bin.yml
@@ -5,20 +5,20 @@ on:
   schedule:
     - cron: '28 7 * * sun'
 
+env:
+  # Latest release: https://www.postgresql.org/ftp/source/
+  LIBPQ_VERSION: "17.0"
+  # Latest release: https://www.openssl.org/source/
+  # Note: 3.3.2 is broken:
+  # https://github.com/openssl/openssl/issues/25366
+  OPENSSL_VERSION: "3.3.1"
+
 jobs:
 
   linux:  # {{{
     runs-on: ubuntu-latest
     if: true
 
-    env:
-      # Latest release: https://www.postgresql.org/ftp/source/
-      LIBPQ_VERSION: "17.0"
-      # Latest release: https://www.openssl.org/source/
-      # Note: 3.3.2 is broken:
-      # https://github.com/openssl/openssl/issues/25366
-      OPENSSL_VERSION: "3.3.1"
-
     strategy:
       fail-fast: false
       matrix:
@@ -97,89 +97,29 @@ jobs:
 
   # }}}
 
-  macos-14:  # {{{
-    runs-on: macos-14
+  macos:  # {{{
+    runs-on: macos-latest
     if: true
 
     strategy:
       fail-fast: false
       matrix:
-        arch: [arm64]
-        pyver: [cp310, cp311, cp312, cp313]
-
-    env:
-      PG_VERSION: "17"
+        arch: [x86_64, arm64]
+        pyver: [cp38, cp39, cp310, cp311, cp312, cp313]
 
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Create the binary package source tree
-        run: python3 ./tools/build/copy_to_binary.py
-
-      - name: Install PostgreSQL on the runner
-        # As of 2024-09-28 postgresql@17 reports an error but it's available
-        # So in a few days we might be able to drop it.
-        run: |
-          brew update
-          brew install gnu-sed postgresql@${PG_VERSION}
-
-      - name: Start PostgreSQL service
-        run: brew services start postgresql@${PG_VERSION}
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
-        with:
-          package-dir: psycopg_binary
-        env:
-          CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}}
-          CIBW_ARCHS_MACOS: ${{matrix.arch}}
-          CIBW_TEST_REQUIRES: ./psycopg[test] ./psycopg_pool
-          CIBW_TEST_COMMAND: >-
-            pytest {project}/tests -m 'not slow and not flakey' --color yes
-          CIBW_ENVIRONMENT: >-
-            MACOSX_DEPLOYMENT_TARGET=14.0
-            PSYCOPG_IMPL=binary
-            PSYCOPG_TEST_DSN='dbname=postgres'
-            PATH="/opt/homebrew/opt/postgresql@${PG_VERSION}/bin:$PATH"
-            PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${PG_VERSION}"
-            PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${PG_VERSION}"
+      - name: Checkout repos
+        uses: actions/checkout@v4
 
-      - uses: actions/upload-artifact@v4
+      - name: Cache libpq build
+        uses: actions/cache@v4
         with:
-          name: macos-14-${{matrix.pyver}}-macosx_${{matrix.arch}}
-          path: ./wheelhouse/*.whl
-
-
-  # }}}
-
-  macos-13:  # {{{
-    runs-on: macos-13
-    if: true
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # These archs require an Apple M1 runner: [arm64, universal2]
-        arch: [x86_64]
-        pyver: [cp38, cp39, cp310, cp311, cp312, cp313]
-
-    env:
-      PG_VERSION: "17"
-
-    steps:
-      - uses: actions/checkout@v4
+          path: /tmp/libpq.build
+          key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }}
 
       - name: Create the binary package source tree
         run: python3 ./tools/build/copy_to_binary.py
 
-      - name: Install PostgreSQL on the runner
-        run: |
-          brew update
-          brew install gnu-sed postgresql@${PG_VERSION}
-
-      - name: Start PostgreSQL service
-        run: brew services start postgresql@${PG_VERSION}
-
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.22.0
         with:
@@ -187,20 +127,24 @@ jobs:
         env:
           CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}}
           CIBW_ARCHS_MACOS: ${{matrix.arch}}
+          MACOSX_ARCHITECTURE: ${{matrix.arch}}
+          CIBW_BEFORE_ALL_MACOS: ./tools/build/wheel_macos_before_all.sh
           CIBW_TEST_REQUIRES: ./psycopg[test] ./psycopg_pool
           CIBW_TEST_COMMAND: >-
             pytest {project}/tests -m 'not slow and not flakey' --color yes
           CIBW_ENVIRONMENT: >-
-            MACOSX_DEPLOYMENT_TARGET=13.0
+            PG_VERSION=17
             PSYCOPG_IMPL=binary
             PSYCOPG_TEST_DSN='dbname=postgres'
-            PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH"
-            PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${PG_VERSION}"
-            PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${PG_VERSION}"
+            LIBPQ_BUILD_PREFIX=/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+            PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${{env.LIBPQ_VERSION}}"
+            PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${{env.LIBPQ_VERSION}}"
 
-      - uses: actions/upload-artifact@v4
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
         with:
-          name: macos-13-${{matrix.pyver}}-macosx_${{matrix.arch}}
+          name: macos-${{matrix.pyver}}-${{matrix.arch}}
           path: ./wheelhouse/*.whl
 
 
@@ -276,12 +220,11 @@ jobs:
 
   # }}}
 
-  merge:
+  merge:  # {{{
     runs-on: ubuntu-latest
     needs:
       - linux
-      - macos-14
-      - macos-13
+      - macos
       - windows
     steps:
       - name: Merge Artifacts
@@ -289,3 +232,5 @@ jobs:
         with:
           name: psycopg-binary-artifact
           delete-merged: true
+
+  # }}}
diff --git a/tools/build/build_libpq.sh b/tools/build/build_libpq.sh
index 5fe98e986..c3711210b 100755
--- a/tools/build/build_libpq.sh
+++ b/tools/build/build_libpq.sh
@@ -1,12 +1,15 @@
 #!/bin/bash
 
-# Build a modern version of libpq and depending libs from source on Centos 5
+# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS
 
 set -euo pipefail
 
 postgres_version="${LIBPQ_VERSION}"
 openssl_version="${OPENSSL_VERSION}"
 
+# Latest release: https://kerberos.org/dist/
+krb5_version="1.21.3"
+
 # Latest release: https://openldap.org/software/download/
 ldap_version="2.6.8"
 
@@ -15,13 +18,28 @@ sasl_version="2.1.28"
 
 export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build}
 
-if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
+case "$(uname)" in
+    Darwin)
+        ID=macos
+        library_suffix=dylib
+        ;;
+
+    Linux)
+        source /etc/os-release
+        library_suffix=so
+        ;;
+
+    *)
+        echo "$0: unexpected Operating system: '$(uname)'" >&2
+        exit 1
+        ;;
+esac
+
+if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.${library_suffix}" ]]; then
     echo "libpq already available: build skipped" >&2
     exit 0
 fi
 
-source /etc/os-release
-
 case "$ID" in
     centos)
         yum update -y
@@ -34,13 +52,46 @@ case "$ID" in
             openssl-dev tzdata zlib-dev
         ;;
 
+    macos)
+        brew install automake m4 libtool
+        # If available, libpq seemingly insists on linking against homebrew's
+        # openssl no matter what so remove it. Since homebrew's curl depends on
+        # it, force use of system curl.
+        brew uninstall --force --ignore-dependencies openssl gettext curl
+        if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then
+            MACOSX_ARCHITECTURE="$(uname -m)"
+        fi
+        # Set the deployment target to be <= to that of the oldest supported Python version.
+        # e.g. https://www.python.org/downloads/release/python-380/
+        if [ "$MACOSX_ARCHITECTURE" == "x86_64" ]; then
+            export MACOSX_DEPLOYMENT_TARGET=10.9
+        else
+            export MACOSX_DEPLOYMENT_TARGET=11.0
+        fi
+        ;;
+
     *)
         echo "$0: unexpected Linux distribution: '$ID'" >&2
         exit 1
         ;;
 esac
 
-if [ "$ID" == "centos" ]; then
+
+if [ "$ID" == "macos" ]; then
+    make_configure_standard_flags=( \
+        --prefix=${LIBPQ_BUILD_PREFIX} \
+        "CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ -arch $MACOSX_ARCHITECTURE" \
+        "LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib -arch $MACOSX_ARCHITECTURE" \
+    )
+else
+    make_configure_standard_flags=( \
+        --prefix=${LIBPQ_BUILD_PREFIX} \
+        CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ \
+        "LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib -L${LIBPQ_BUILD_PREFIX}/lib64" \
+    )
+fi
+
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
   if [[ ! -f "${LIBPQ_BUILD_PREFIX}/openssl.cnf" ]]; then
 
     # Build openssl if needed
@@ -53,8 +104,14 @@ if [ "$ID" == "centos" ]; then
 
         pushd "${openssl_dir}"
 
-        ./config --prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \
-            zlib -fPIC shared
+        options=(--prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \
+            zlib -fPIC shared)
+        if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then
+            ./config $options
+        else
+            ./configure "darwin64-$MACOSX_ARCHITECTURE-cc" $options
+        fi
+
         make depend
         make
     else
@@ -69,8 +126,29 @@ if [ "$ID" == "centos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ]; then
-  if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libsasl2.so" ]]; then
+if [ "$ID" == "macos" ]; then
+
+    # Build kerberos if needed
+    krb5_dir="krb5-${krb5_version}/src"
+    if [ ! -d "${krb5_dir}" ]; then
+        curl -fsSL "https://kerberos.org/dist/krb5/${krb5_version%.*}/krb5-${krb5_version}.tar.gz" \
+            | tar xzf -
+
+        pushd "${krb5_dir}"
+        ./configure "${make_configure_standard_flags[@]}"
+        make
+    else
+        pushd "${krb5_dir}"
+    fi
+
+    make install
+    popd
+
+fi
+
+
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
+  if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libsasl2.${library_suffix}" ]]; then
 
     # Build libsasl2 if needed
     # The system package (cyrus-sasl-devel) causes an amazing error on i686:
@@ -86,8 +164,7 @@ if [ "$ID" == "centos" ]; then
         pushd "${sasl_dir}"
 
         autoreconf -i
-        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
-            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework
         make
     else
         pushd "${sasl_dir}"
@@ -103,8 +180,8 @@ if [ "$ID" == "centos" ]; then
 fi
 
 
-if [ "$ID" == "centos" ]; then
-  if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libldap.so" ]]; then
+if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then
+  if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libldap.${library_suffix}" ]]; then
 
     # Build openldap if needed
     ldap_tag="${ldap_version}"
@@ -116,8 +193,7 @@ if [ "$ID" == "centos" ]; then
 
         pushd "${ldap_dir}"
 
-        ./configure --prefix=${LIBPQ_BUILD_PREFIX} --enable-backends=no --enable-null \
-            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        ./configure "${make_configure_standard_flags[@]}" --enable-backends=no --enable-null
 
         make depend
         make -C libraries/liblutil/
@@ -131,7 +207,7 @@ if [ "$ID" == "centos" ]; then
     make -C libraries/liblber/ install
     make -C libraries/libldap/ install
     make -C include/ install
-    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so*
+    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}*
     popd
 
   fi
@@ -148,19 +224,19 @@ if [ ! -d "${postgres_dir}" ]; then
 
     pushd "${postgres_dir}"
 
-    # Match the default unix socket dir default with what defined on Ubuntu and
-    # Red Hat, which seems the most common location
-    sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
+    if [ "$ID" != "macos" ]; then
+        # Match the default unix socket dir default with what defined on Ubuntu and
+        # Red Hat, which seems the most common location
+        sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
 '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \
-        src/include/pg_config_manual.h
+            src/include/pg_config_manual.h
+    fi
 
     export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib:${LIBPQ_BUILD_PREFIX}/lib64"
 
-    ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \
-        --without-readline --without-icu \
+    ./configure "${make_configure_standard_flags[@]}" --sysconfdir=/etc/postgresql-common \
         --with-gssapi --with-openssl --with-pam --with-ldap \
-        CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ \
-        LDFLAGS="-L${LIBPQ_BUILD_PREFIX}/lib -L${LIBPQ_BUILD_PREFIX}/lib64"
+        --without-readline --without-icu
     make -C src/interfaces/libpq
     make -C src/bin/pg_config
     make -C src/include
@@ -174,4 +250,4 @@ make -C src/bin/pg_config install
 make -C src/include install
 popd
 
-find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \;
+find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \;
diff --git a/tools/build/wheel_macos_before_all.sh b/tools/build/wheel_macos_before_all.sh
new file mode 100755
index 000000000..e3fb2f437
--- /dev/null
+++ b/tools/build/wheel_macos_before_all.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Configure the environment needed to build wheel packages on Mac OS.
+# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_MACOS
+#
+# The PG_VERSION env var must be set to a Postgres major version (e.g. 16).
+
+set -euo pipefail
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
+
+# Build dependency libraries
+"${prjdir}/tools/build/build_libpq.sh"
+
+# Show dependency tree
+otool -L /tmp/libpq.build/lib/*.dylib
+
+brew install gnu-sed postgresql@${PG_VERSION}
+brew link --overwrite postgresql@${PG_VERSION}
+
+# Start the database for testing
+brew services start postgresql@${PG_VERSION}
+
+# Wait for postgres to come up
+for i in $(seq 10 -1 0); do
+  eval pg_isready && break
+  if [ $i == 0 ]; then
+      echo "PostgreSQL service not ready, giving up"
+      exit 1
+  fi
+  echo "PostgreSQL service not ready, waiting a bit, attempts left: $i"
+  sleep 5
+done