From: Daniele Varrazzo Date: Sat, 17 Jul 2021 00:39:40 +0000 (+0200) Subject: Test C extension on Windows and build binary packages X-Git-Tag: 3.0.dev1~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6d930d6e8098d056d00e386b773d29acf74fdebd;p=thirdparty%2Fpsycopg.git Test C extension on Windows and build binary packages The built wheel package seems broken, but delocating it seems to fix it. Use the PG_CONFIG env var to specify the pg_config location because hacking the path is not straightforward with cibuildwheel and has unwanted side effects with delvewheel. --- diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index bfeade2b1..54e5269cf 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -156,9 +156,6 @@ jobs: windows: name: Build Windows packages runs-on: windows-2019 - # TODO: Currently disabled: not completed yet. See the - # wheel_windows_before_all script. - if: false strategy: matrix: @@ -169,12 +166,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Start PostgreSQL service for test - run: | - $postgreSqlSvc = Get-Service "postgresql*" - Set-Service $postgreSqlSvc.Name -StartupType manual - $postgreSqlSvc.Start() - - name: Create the binary package source tree run: python3 ./tools/build/copy_to_binary.py @@ -191,10 +182,11 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- delvewheel repair -w {dest_dir} {wheel} CIBW_BEFORE_TEST: pip install ./psycopg[test] - CIBW_TEST_COMMAND: pytest {project}/tests -m 'not slow' --color yes - CIBW_ENVIRONMENT: >- + CIBW_TEST_COMMAND: pytest {project}/tests -m "not slow" --color yes + CIBW_ENVIRONMENT_WINDOWS: >- PSYCOPG_IMPL=binary - PSYCOPG_TEST_DSN='host=172.17.0.1 user=postgres' + PSYCOPG_TEST_DSN='host=127.0.0.1 user=postgres' + PG_CONFIG='C:\Program Files\PostgreSQL\13\bin\pg_config.exe' - uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e597e81c..9aa107db3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -117,18 +117,19 @@ jobs: windows: name: Test on Windows runs-on: windows-2019 + strategy: fail-fast: false matrix: include: - - {impl: c, python: 3.6} - - {impl: c, python: 3.7} - - {impl: c, python: 3.8} - - {impl: c, python: 3.9} - {impl: python, python: 3.6} - {impl: python, python: 3.7} - {impl: python, python: 3.8} - {impl: python, python: 3.9} + - {impl: c, python: 3.6} + - {impl: c, python: 3.7} + - {impl: c, python: 3.8} + - {impl: c, python: 3.9} env: PSYCOPG_IMPL: ${{ matrix.impl }} @@ -141,51 +142,43 @@ jobs: - name: Start PostgreSQL service for test run: | - $postgreSqlSvc = Get-Service "postgresql*" - Set-Service $postgreSqlSvc.Name -StartupType manual - $postgreSqlSvc.Start() + $PgSvc = Get-Service "postgresql*" + Set-Service $PgSvc.Name -StartupType manual + $PgSvc.Start() - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - name: Install tox - run: pip install tox + run: pip install tox wheel - name: Run tests (Python implementation) - run: tox -c psycopg -e ${{ matrix.python }} -- --color yes if: ${{ matrix.impl == 'python' }} + run: tox -c psycopg -e ${{ matrix.python }} -- --color yes - # An attepnt to add the path to find libpq.lib and avoid: - # "The specified module could not be found" - # It doesn't work, but leaving this here because appending to path - # is a pretty clunky matter I don't want to google for again. - - name: append the libpq.lib path to the PATH - run: >- - echo "C:\Program Files\PostgreSQL\13\lib" | - Out-File -FilePath $env:GITHUB_PATH -Append + # Build a wheel package of the C extensions. + # Specify a pg_config explicitly because there are wrong ones along the + # path (https://github.com/actions/virtual-environments/issues/3730). + # + # Don't add the Postgres bindir to the path because that causes problems + # in delocating (https://github.com/pypa/cibuildwheel/issues/766). + # + # If the wheel is not delocated, import fails with some dll not found + # (but it won't tell which one). + - name: Build the C wheel if: ${{ matrix.impl == 'c' }} - - - name: Run tests (C implementation) - run: tox -c psycopg_c -e ${{ matrix.python }} -- --color yes - - # For the moment don't run the C module. It compiles alright - # but import fails with: - # - # error importing requested 'c' wrapper: DLL load failed while - # importing pq: The specified module could not be found - # if: ${{ matrix.impl == 'c' }} - if: false - - env: - # the pg_config returns the wrong path - # https://github.com/actions/runner/issues/1178 - # Note: this env var name must be in the tox.ini passenv - PG_LIBPATH: 'C:\Program Files\PostgreSQL\13\lib' - - # TODO Remove once testing is fixed - - name: Build the C extension - run: python ./psycopg_c/setup.py build + run: | + $env:PG_CONFIG = "C:\Program Files\PostgreSQL\13\bin\pg_config.exe" + python ./psycopg_c/setup.py bdist_wheel + pip install delvewheel + &"delvewheel" repair @(Get-ChildItem psycopg_c\dist\*.whl) + + # tox will only build the package from sdist, not from wheel, so we can't + # use it for testing. Just test everything in the global installation. + - name: Install and run tests (C implementation) if: ${{ matrix.impl == 'c' }} - env: - PG_LIBPATH: 'C:\Program Files\PostgreSQL\13\lib' + run: | + pip install ./psycopg/[test] + &"pip" install @(Get-ChildItem wheelhouse\*.whl) + pytest --color yes diff --git a/psycopg_c/setup.py b/psycopg_c/setup.py index fb3163b84..6e131db5f 100644 --- a/psycopg_c/setup.py +++ b/psycopg_c/setup.py @@ -29,10 +29,13 @@ with open("psycopg_c/version.py") as f: def get_config(what: str) -> str: + # Allow to specify PG_CONFIG using an env var + # Changing the path in the cibuildwheel image seems difficult + pg_config = os.environ.get("PG_CONFIG", "pg_config") try: - out = sp.run(["pg_config", f"--{what}"], stdout=sp.PIPE, check=True) + out = sp.run([pg_config, f"--{what}"], stdout=sp.PIPE, check=True) except Exception as e: - log.error("cannot build C module: %s", e) + log.error(f"couldn't run {pg_config!r} --{what}: %s", e) raise else: return out.stdout.strip().decode("utf8") @@ -58,12 +61,6 @@ class psycopg_build_ext(build_ext): ext.include_dirs.append(includedir) ext.library_dirs.append(libdir) - # hack to build on GH Actions (pg_config --libdir broken) - # https://github.com/actions/runner/issues/1178 - for path in os.environ.get("PG_LIBPATH", "").split(os.pathsep): - if path: - ext.library_dirs.append(path) - if sys.platform == "win32": # For __imp_htons and others ext.libraries.append("ws2_32") diff --git a/tools/build/wheel_windows_before_all.ps1 b/tools/build/wheel_windows_before_all.ps1 index e93ab3862..e7bffe3d3 100755 --- a/tools/build/wheel_windows_before_all.ps1 +++ b/tools/build/wheel_windows_before_all.ps1 @@ -1,34 +1,4 @@ -#!/bin/bash - -# Configure the environment needed to build wheel packages on Windows. -# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_WINDOWS - -# The script is currently incomplete. The image seems to have postgres -# installed but without libpq.lib, so installation fails. This is an incomplete -# attempt to download the binary package (which is 235MB and takes forever) -# and use it for building. - -# Set-PSDebug -Trace 1 - -python -c "import os; print(os.environ['PATH'])" - -# choco install postgresql13 --params '/Password:password' - -# From: https://www.enterprisedb.com/download-postgresql-binaries -Invoke-WebRequest ` - -Uri "https://sbp.enterprisedb.com/getfile.jsp?fileid=1257716" ` - -OutFile C:\postgresql-13.3-2-windows-x64-binaries.zip - -Expand-Archive ` - -LiteralPath C:\postgresql-13.3-2-windows-x64-binaries.zip ` - -DestinationPath C:\ - -# python -c "import os; print(os.environ['PATH'])" - -# pg_config - -# dir C:/STRAWB~1/c/bin -# dir C:/STRAWB~1/c/lib - -dir C:\pgsql\bin -C:\pgsql\bin\pg_config +# Start PostgreSQL service for test +$PgSvc = Get-Service "postgresql*" +Set-Service $PgSvc.Name -StartupType manual +$PgSvc.Start()