From 4735eb02ea45ef5540cac644363544d7e0570d7d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Apr 2024 01:57:50 +0200 Subject: [PATCH] ci(macos): test and build macOS packages on M1 runners Separate macos runners because: The macos-14 runner can build amd64 images, but doesn't have Python < 3.10. The macos-12 runner can only build x86_64 images. Run Postgres from CI rather than from cibuildwheel, as cibw won't use a docker image on macOS anyway. It makes it more uniform w.r.t. other runners and doesn't require a "before" script. --- .github/workflows/packages-bin.yml | 70 ++++++++++++++++++++++++--- .github/workflows/tests.yml | 69 +++++++++++++++++++++++--- tools/build/wheel_macos_before_all.sh | 26 ---------- 3 files changed, 124 insertions(+), 41 deletions(-) delete mode 100755 tools/build/wheel_macos_before_all.sh diff --git a/.github/workflows/packages-bin.yml b/.github/workflows/packages-bin.yml index ec5e1757a..c4115b7da 100644 --- a/.github/workflows/packages-bin.yml +++ b/.github/workflows/packages-bin.yml @@ -92,8 +92,57 @@ jobs: # }}} - macos: # {{{ - runs-on: macos-latest + macos-14: # {{{ + runs-on: macos-14 + if: true + + strategy: + fail-fast: false + matrix: + arch: [arm64] + pyver: [cp310, cp311, cp312] + + env: + PG_VERSION: "16" + + steps: + - uses: actions/checkout@v4 + + - name: Create the binary package source tree + run: python3 ./tools/build/copy_to_binary.py + + - name: Install PostgreSQL on the runner + run: brew install gnu-sed postgresql@${PG_VERSION} + + - name: Start PostgreSQL service + run: brew services start postgresql@${PG_VERSION} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + with: + package-dir: psycopg_binary + env: + CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}} + CIBW_ARCHS_MACOS: ${{matrix.arch}} + CIBW_TEST_REQUIRES: ./psycopg[test] ./psycopg_pool + CIBW_TEST_COMMAND: >- + pytest {project}/tests -m 'not slow and not flakey' --color yes + CIBW_ENVIRONMENT: >- + PSYCOPG_IMPL=binary + PSYCOPG_TEST_DSN='dbname=postgres' + PATH="/opt/homebrew/opt/postgresql@${PG_VERSION}/bin:$PATH" + PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${PG_VERSION}" + PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${PG_VERSION}" + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + + # }}} + + macos-12: # {{{ + runs-on: macos-12 if: true strategy: @@ -103,30 +152,37 @@ jobs: arch: [x86_64] pyver: [cp38, cp39, cp310, cp311, cp312] + env: + PG_VERSION: "16" + steps: - uses: actions/checkout@v4 - name: Create the binary package source tree run: python3 ./tools/build/copy_to_binary.py + - name: Install PostgreSQL on the runner + run: brew install gnu-sed postgresql@${PG_VERSION} + + - name: Start PostgreSQL service + run: brew services start postgresql@${PG_VERSION} + - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 with: package-dir: psycopg_binary env: CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}} - CIBW_ARCHS_MACOS: x86_64 - CIBW_BEFORE_ALL_MACOS: ./tools/build/wheel_macos_before_all.sh + CIBW_ARCHS_MACOS: ${{matrix.arch}} CIBW_TEST_REQUIRES: ./psycopg[test] ./psycopg_pool CIBW_TEST_COMMAND: >- pytest {project}/tests -m 'not slow and not flakey' --color yes CIBW_ENVIRONMENT: >- - PG_VERSION=16 PSYCOPG_IMPL=binary PSYCOPG_TEST_DSN='dbname=postgres' PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" - PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= $PG_VERSION" - PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= $PG_VERSION" + PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= ${PG_VERSION}" + PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= ${PG_VERSION}" - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e36712042..94a29e67a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -136,21 +136,17 @@ jobs: # }}} - macos: # {{{ - runs-on: macos-latest + macos-14: # {{{ + runs-on: macos-14 if: true strategy: fail-fast: false matrix: include: - - {impl: python, python: "3.8"} - - {impl: python, python: "3.9"} - {impl: python, python: "3.10"} - {impl: python, python: "3.11"} - {impl: python, python: "3.12"} - - {impl: c, python: "3.8"} - - {impl: c, python: "3.9"} - {impl: c, python: "3.10"} - {impl: c, python: "3.11"} - {impl: c, python: "3.12"} @@ -163,6 +159,63 @@ jobs: # Don't run timing-based tests as they regularly fail. # pproxy-based tests fail too, with the proxy not coming up in 2s. NOT_MARKERS: "timing proxy mypy" + PG_VERSION: "16" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install PostgreSQL on the runner + run: brew install postgresql@${PG_VERSION} + + - name: Start PostgreSQL service + run: brew services start postgresql@${PG_VERSION} + + - name: Find the libpq + if: ${{ matrix.impl == 'python' }} + run: | + echo "DYLD_LIBRARY_PATH=/opt/homebrew/opt/postgresql@${PG_VERSION}/lib:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Include psycopg-c to the packages to install + if: ${{ matrix.impl == 'c' }} + run: | + echo "DEPS=$DEPS ./psycopg_c" >> $GITHUB_ENV + echo "PATH=/opt/homebrew/opt/postgresql@${PG_VERSION}/bin:$PATH" >> $GITHUB_ENV + + - name: Install Python packages + run: pip install $DEPS + + - name: Run tests + run: ./tools/build/ci_test.sh + + + # }}} + + macos-12: # {{{ + runs-on: macos-12 + if: true + + strategy: + fail-fast: false + matrix: + include: + - {impl: python, python: "3.8"} + - {impl: python, python: "3.9"} + - {impl: c, python: "3.8"} + - {impl: c, python: "3.9"} + + env: + PSYCOPG_IMPL: ${{ matrix.impl }} + DEPS: ./psycopg[test] ./psycopg_pool + PSYCOPG_TEST_DSN: "host=127.0.0.1 user=runner dbname=postgres" + # MacOS on GitHub Actions seems particularly slow. + # Don't run timing-based tests as they regularly fail. + # pproxy-based tests fail too, with the proxy not coming up in 2s. + NOT_MARKERS: "timing proxy mypy" + PG_VERSION: "16" steps: - uses: actions/checkout@v4 @@ -172,10 +225,10 @@ jobs: python-version: ${{ matrix.python }} - name: Install PostgreSQL on the runner - run: brew install postgresql@14 + run: brew install postgresql@${PG_VERSION} - name: Start PostgreSQL service - run: brew services start postgresql + run: brew services start postgresql@${PG_VERSION} - name: Include psycopg-c to the packages to install if: ${{ matrix.impl == 'c' }} diff --git a/tools/build/wheel_macos_before_all.sh b/tools/build/wheel_macos_before_all.sh deleted file mode 100755 index 051f5ff0c..000000000 --- a/tools/build/wheel_macos_before_all.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Configure the environment needed to build wheel packages on Mac OS. -# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_MACOS -# -# The PG_VERSION env var must be set to a Postgres major version (e.g. 16). - -set -euo pipefail - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -brew install gnu-sed postgresql@${PG_VERSION} - -# Start the database for testing -brew services start postgresql - -# Wait for postgres to come up -for i in $(seq 10 -1 0); do - eval pg_isready && break - if [ $i == 0 ]; then - echo "PostgreSQL service not ready, giving up" - exit 1 - fi - echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" - sleep 5 -done -- 2.47.2