ci: uniform and clean up test workflow

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index cd263bdab5f22c102c624b5e316dfc4cee8c5575..94505cedb3d60a4bc99285de0df0b17b13661fbd 100644 (file)
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -35,7 +35,7 @@ jobs:
           - {impl: c, python: "3.7", postgres: "postgres:15", libpq: newest}
           - {impl: c, python: "3.8", postgres: "postgres:13"}
           - {impl: c, python: "3.9", postgres: "postgres:14"}
-          - {impl: c, python: "3.10", postgres: "postgres:13", libpq: oldest}
+          - {impl: c, python: "3.10", postgres: "postgres:11", libpq: oldest}
           - {impl: c, python: "3.11", postgres: "postgres:10", libpq: newest}
 
           - {impl: python, python: "3.9", ext: dns, postgres: "postgres:14"}
@@ -44,8 +44,7 @@ jobs:
     env:
       PSYCOPG_IMPL: ${{ matrix.impl }}
       DEPS: ./psycopg[test] ./psycopg_pool
-      PSYCOPG_TEST_DSN: "host=127.0.0.1 user=postgres"
-      PGPASSWORD: password
+      PSYCOPG_TEST_DSN: "host=127.0.0.1 user=postgres password=password"
       MARKERS: ""
 
       # Enable to run tests using the minimum version of dependencies.
@@ -58,7 +57,7 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Run PostgreSQL
+      - name: Start PostgreSQL service
         # Note: this would love to be a service, but I don't see a way to pass
         # the args to the docker run command line.
         run: |
@@ -67,51 +66,27 @@ jobs:
             -e POSTGRES_PASSWORD=password ${{ matrix.postgres }} \
             -c max_prepared_transactions=10
 
-      - name: Install the newest libpq version available
-        if: ${{ matrix.libpq == 'newest' }}
-        run: |
-          set -x
-
-          curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
-            | gpg --dearmor \
-            | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null
-
-          # NOTE: in order to test with a preview release, add its number to
-          # the deb entry. For instance, to test on preview Postgres 16, use:
-          # "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main 16"
-          rel=$(lsb_release -c -s)
-          echo "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main" \
-            | sudo tee -a /etc/apt/sources.list.d/pgdg.list > /dev/null
-          sudo apt-get -qq update
-
-          pqver=$(apt-cache show libpq5  | grep ^Version: | head -1 \
-            | awk '{print $2}')
-          sudo apt-get -qq -y install "libpq-dev=${pqver}" "libpq5=${pqver}"
-
-      - name: Install the oldest libpq version available
-        if: ${{ matrix.libpq == 'oldest' }}
+      - name: Install the wanted libpq version
+        run: ./tools/build/ci_install_libpq.sh ${{ matrix.libpq }}
+
+      - name: Include psycopg-c to the packages to install
+        if: ${{ matrix.impl == 'c' }}
         run: |
-          set -x
-          pqver=$(apt-cache show libpq5  | grep ^Version: | tail -1 \
-            | awk '{print $2}')
-          sudo apt-get -qq -y --allow-downgrades install \
-            "libpq-dev=${pqver}" "libpq5=${pqver}"
+          echo "DEPS=$DEPS ./psycopg_c" >> $GITHUB_ENV
 
-      - if: ${{ matrix.ext == 'dns' }}
+      - name: Include dnspython to the packages to install
+        if: ${{ matrix.ext == 'dns' }}
         run: |
           echo "DEPS=$DEPS dnspython" >> $GITHUB_ENV
           echo "MARKERS=$MARKERS dns" >> $GITHUB_ENV
 
-      - if: ${{ matrix.ext == 'postgis' }}
+      - name: Include shapely to the packages to install
+        if: ${{ matrix.ext == 'postgis' }}
         run: |
           echo "DEPS=$DEPS shapely" >> $GITHUB_ENV
           echo "MARKERS=$MARKERS postgis" >> $GITHUB_ENV
 
-      - if: ${{ matrix.impl == 'c' }}
-        run: |
-          echo "DEPS=$DEPS ./psycopg_c" >> $GITHUB_ENV
-
-      - name: Install Python dependencies
+      - name: Install Python packages
         run: pip install $DEPS
 
       - name: Run tests
@@ -147,29 +122,29 @@ jobs:
       # Don't run timing-based tests as they regularly fail.
       # pproxy-based tests fail too, with the proxy not coming up in 2s.
       NOT_MARKERS: "timing proxy mypy"
+
+      # Enable to run tests using the minimum version of dependencies.
       # PIP_CONSTRAINT: ${{ github.workspace }}/tests/constraints.txt
 
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
       - name: Install PostgreSQL on the runner
         run: brew install postgresql@14
 
-      - name: Start PostgreSQL service for test
+      - name: Start PostgreSQL service
         run: brew services start postgresql
 
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-
-      - if: ${{ matrix.impl == 'c' }}
-        # skip tests failing on importing psycopg_c.pq on subprocess
-        # they only fail on Travis, work ok locally under tox too.
-        # TODO: check the same on GitHub Actions
+      - name: Include psycopg-c to the packages to install
+        if: ${{ matrix.impl == 'c' }}
         run: |
           echo "DEPS=$DEPS ./psycopg_c" >> $GITHUB_ENV
 
-      - name: Install Python dependencies
+      - name: Install Python packages
         run: pip install $DEPS
 
       - name: Run tests
@@ -203,40 +178,50 @@ jobs:
       PSYCOPG_TEST_DSN: "host=127.0.0.1 dbname=postgres"
       # On windows pproxy doesn't seem very happy. Also a few timing test fail.
       NOT_MARKERS: "timing proxy mypy"
+
+      # Enable to run tests using the minimum version of dependencies.
       # PIP_CONSTRAINT: ${{ github.workspace }}/tests/constraints.txt
 
+    defaults:
+      run:
+        shell: bash
+
     steps:
       - uses: actions/checkout@v3
 
-      - name: Start PostgreSQL service for test
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Start PostgreSQL service
         run: |
           $PgSvc = Get-Service "postgresql*"
           Set-Service $PgSvc.Name -StartupType manual
           $PgSvc.Start()
+        shell: pwsh
 
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-
-      # Build a wheel package of the C extensions.
-      # If the wheel is not delocated, import fails with some dll not found
-      # (but it won't tell which one).
       - name: Build the C wheel
         if: ${{ matrix.impl == 'c' }}
         run: |
           pip install delvewheel wheel
-          $env:Path = "C:\Program Files\PostgreSQL\14\bin\;$env:Path"
-          python ./psycopg_c/setup.py bdist_wheel
-          &"delvewheel" repair `
-            --no-mangle "libiconv-2.dll;libwinpthread-1.dll" `
-            @(Get-ChildItem psycopg_c\dist\*.whl)
-          &"pip" install @(Get-ChildItem wheelhouse\*.whl)
+
+          # The windows runner is a total mess, with random copies of the libpq
+          # scattered all over the places. Give precedence to the one under our
+          # control (or the illusion of it).
+          export PATH="/c/Program Files/PostgreSQL/14/bin/:$PATH"
+
+          # If the wheel is not delocated, import fails with some dll not found
+          # (but it won't tell which one).
+          pip wheel -v -w ./psycopg_c/dist/ ./psycopg_c/
+          delvewheel repair --no-mangle "libiconv-2.dll;libwinpthread-1.dll" \
+            -w ./wheelhouse/ psycopg_c/dist/*.whl
+          echo "DEPS=$DEPS $(ls ./wheelhouse/*.whl)" >> $GITHUB_ENV
+
+      - name: Install Python packages
+        run: pip install $DEPS
 
       - name: Run tests
-        run: |
-          pip install $DEPS
-          ./tools/build/ci_test.sh
-        shell: bash
+        run: ./tools/build/ci_test.sh
 
 
   # }}}
@@ -263,7 +248,7 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Run CockroachDB
+      - name: Start CockroachDB service
         # Note: this would love to be a service, but I don't see a way to pass
         # the args to the docker run command line.
         run: |
@@ -271,39 +256,19 @@ jobs:
           docker run --rm -d --name crdb -p 26257:26257 \
             cockroachdb/cockroach:${{ matrix.crdb }} start-single-node --insecure
 
-      - name: Install the newest libpq version available
-        if: ${{ matrix.libpq == 'newest' }}
-        run: |
-          set -x
-
-          curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
-            | gpg --dearmor \
-            | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null
-
-          # NOTE: in order to test with a preview release, add its number to
-          # the deb entry. For instance, to test on preview Postgres 16, use:
-          # "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main 16"
-          rel=$(lsb_release -c -s)
-          echo "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main" \
-            | sudo tee -a /etc/apt/sources.list.d/pgdg.list > /dev/null
-          sudo apt-get -qq update
-
-          pqver=$(apt-cache show libpq5  | grep ^Version: | head -1 \
-            | awk '{print $2}')
-          sudo apt-get -qq -y install "libpq-dev=${pqver}" "libpq5=${pqver}"
+      - name: Install the wanted libpq version
+        run: ./tools/build/ci_install_libpq.sh ${{ matrix.libpq }}
 
-      - if: ${{ matrix.impl == 'c' }}
+      - name: Include psycopg-c to the packages to install
+        if: ${{ matrix.impl == 'c' }}
         run: |
           echo "DEPS=$DEPS ./psycopg_c" >> $GITHUB_ENV
 
-      - name: Install Python dependencies
+      - name: Install Python packages
         run: pip install $DEPS
 
       - name: Run tests
         run: ./tools/build/ci_test.sh
 
-      - name: Stop CockroachDB
-        run: docker kill crdb
-
 
   # }}}

diff --git a/tools/build/ci_install_libpq.sh b/tools/build/ci_install_libpq.sh
new file mode 100755 (executable)
index 0000000..64b7824
--- /dev/null
+++ b/tools/build/ci_install_libpq.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Install the desired libpw in github action (Linux runner)
+#
+# Specify `oldest` or `newest` as first argument in order to choose the oldest
+# available to the debian distro or the newest available from the pgdg ppa.
+
+set -euo pipefail
+set -x
+
+libpq=${1:-}
+
+case "$libpq" in
+    "")
+        # Assume a libpq is already installed in the system. We don't care about
+        # the version.
+        exit 0
+        ;;
+
+    oldest)
+        pqver=$(apt-cache show libpq5 | grep ^Version: | tail -1 | awk '{print $2}')
+        sudo apt-get -qq -y --allow-downgrades install \
+            "libpq-dev=${pqver}" "libpq5=${pqver}"
+        ;;
+
+    newest)
+        curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
+            | gpg --dearmor \
+            | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null
+
+        # NOTE: in order to test with a preview release, add its number to
+        # the deb entry. For instance, to test on preview Postgres 16, use:
+        # "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main 16"
+        rel=$(lsb_release -c -s)
+        echo "deb http://apt.postgresql.org/pub/repos/apt ${rel}-pgdg main" \
+            | sudo tee -a /etc/apt/sources.list.d/pgdg.list > /dev/null
+        sudo apt-get -qq update
+
+        pqver=$(apt-cache show libpq5 | grep ^Version: | head -1 | awk '{print $2}')
+        sudo apt-get -qq -y install "libpq-dev=${pqver}" "libpq5=${pqver}"
+        ;;
+
+    *)
+        echo "Unexpected wanted libpq: '${libpq}'" >&2
+        exit 1
+        ;;
+
+esac