From: Trim21 Date: Sun, 15 Dec 2024 22:38:08 +0000 (+0800) Subject: ci: use vcpkg to install libpq on windows X-Git-Tag: 3.2.4~15^2~1 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=c7a30776177fe7411c4500462349103f90d59d90;p=thirdparty%2Fpsycopg.git ci: use vcpkg to install libpq on windows --- diff --git a/.github/workflows/packages-bin.yml b/.github/workflows/packages-bin.yml index 316e4c633..d79981f60 100644 --- a/.github/workflows/packages-bin.yml +++ b/.github/workflows/packages-bin.yml @@ -218,6 +218,10 @@ jobs: pyver: [cp38, cp39, cp310, cp311, cp312, cp313] steps: + # there are some other libpq in PATH + - run: rm -rf c:/tools/php C:/Strawberry/c/bin + shell: bash + - uses: actions/checkout@v4 - name: Start PostgreSQL service for test @@ -226,6 +230,16 @@ jobs: Set-Service $PgSvc.Name -StartupType manual $PgSvc.Start() + - name: Export GitHub Actions cache environment variables + uses: actions/github-script@v7 + with: + script: | + const path = require('path') + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + core.addPath(path.join(process.env.VCPKG_INSTALLATION_ROOT, 'installed/x64-windows-release/lib')); + core.addPath(path.join(process.env.VCPKG_INSTALLATION_ROOT, 'installed/x64-windows-release/bin')); + - name: Create the binary package source tree run: python3 ./tools/build/copy_to_binary.py @@ -234,6 +248,7 @@ jobs: with: package-dir: psycopg_binary env: + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" # cache vcpkg CIBW_BUILD: ${{matrix.pyver}}-${{matrix.arch}} CIBW_ARCHS_WINDOWS: AMD64 x86 CIBW_BEFORE_BUILD_WINDOWS: '.\tools\build\wheel_win32_before_build.bat' @@ -245,7 +260,6 @@ jobs: pytest {project}/tests -m "not slow and not flakey" --color yes CIBW_ENVIRONMENT_WINDOWS: >- PSYCOPG_IMPL=binary - PATH="C:\\Program Files\\PostgreSQL\\14\\bin;$PATH" PSYCOPG_TEST_DSN="host=127.0.0.1 user=postgres" PSYCOPG_TEST_WANT_LIBPQ_BUILD=">= 16" PSYCOPG_TEST_WANT_LIBPQ_IMPORT=">= 16" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7735bf779..73d9fa092 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -306,6 +306,9 @@ jobs: shell: bash steps: + # there are some extra libpq.dll in PATH + - run: rm -rf c:/tools/php C:/Strawberry/c/bin + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -329,16 +332,29 @@ jobs: if: ${{ github.event_name == 'schedule' }} run: echo "NOT_MARKERS=$NOT_MARKERS refcount" >> $GITHUB_ENV + - name: Export GitHub Actions cache environment variables + # https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-actions-cache + uses: actions/github-script@v7 + with: + script: | + const path = require('path') + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + core.addPath(path.join(process.env.VCPKG_INSTALLATION_ROOT, 'installed/x64-windows-release/lib')); + core.addPath(path.join(process.env.VCPKG_INSTALLATION_ROOT, 'installed/x64-windows-release/bin')); + + - name: Install libpq from vcpkg and install pg_config.exe stub + run: .\tools\build\wheel_win32_before_build.bat + shell: powershell + env: + # cache vcpkg + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" + - name: Build the C wheel if: ${{ matrix.impl == 'c' }} run: | pip install delvewheel wheel - # The windows runner is a total mess, with random copies of the libpq - # scattered all over the places. Give precedence to the one under our - # control (or the illusion of it). - export PATH="/c/Program Files/PostgreSQL/14/bin/:$PATH" - # If the wheel is not delocated, import fails with some dll not found # (but it won't tell which one). pip wheel -v -w ./psycopg_c/dist/ ./psycopg_c/ diff --git a/tools/build/pg_config_vcpkg_stub/pg_config_vcpkg_stub/__init__.py b/tools/build/pg_config_vcpkg_stub/pg_config_vcpkg_stub/__init__.py new file mode 100644 index 000000000..d1cb0cc07 --- /dev/null +++ b/tools/build/pg_config_vcpkg_stub/pg_config_vcpkg_stub/__init__.py @@ -0,0 +1,51 @@ +""" +We use vcpkg in github actions to build psycopg-binary. + +This is a stub to work as `pg_config --libdir` or `pg_config --includedir` to make it work with vcpkg. + +You will need install vcpkg and set `VCPKG_ROOT `env +and run `vcpkg install libpq:x64-windows-release` before you use this script +""" + +import os +import pathlib +import sys +import platform + + +def main(): + # only x64-windows + if not (sys.platform == "win32" and platform.machine() == "AMD64"): + raise Exception("this script should only be used in x64-windows") + + what = sys.argv[1] + + if what == "--help": + print(__doc__) + return + + # on github actions it's `VCPKG_INSTALLATION_ROOT` + if "VCPKG_INSTALLATION_ROOT" not in os.environ: + print("failed to find VCPKG ROOT path", file=sys.stderr) + sys.exit(1) + + vcpkg_root = pathlib.Path(os.environ["VCPKG_INSTALLATION_ROOT"]) + vcpkg_platform_root = vcpkg_root.joinpath("installed/x64-windows-release").resolve() + if not vcpkg_root.joinpath("packages/libpq_x64-windows-release").exists(): + print("libpq not installed with vcpkg", file=sys.stderr) + sys.exit(1) + + if what == "--libdir": + print(str(vcpkg_platform_root.joinpath("lib"))) + return + if what == "--includedir": + print(str(vcpkg_platform_root.joinpath("include"))) + return + + print( + "unexpected command: {!r}\n this maybe out-of-sync between 'psycopg_c/setup.py' and 'tools/build/pg_config_vcpkg_stub/pg_config_vcpkg_stub/__init__.py'".format( + sys.argv[1:] + ), + file=sys.stderr, + ) + sys.exit(1) diff --git a/tools/build/pg_config_vcpkg_stub/pyproject.toml b/tools/build/pg_config_vcpkg_stub/pyproject.toml new file mode 100644 index 000000000..f776905f2 --- /dev/null +++ b/tools/build/pg_config_vcpkg_stub/pyproject.toml @@ -0,0 +1,11 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = 'pg_config_vcpkg_stub' +version = "0" +description = "see docs string in pg_config_vcpkg_stub for more details" + +[project.scripts] +pg_config = 'pg_config_vcpkg_stub:main' diff --git a/tools/build/wheel_win32_before_build.bat b/tools/build/wheel_win32_before_build.bat index 1d24ab8db..5bd5dc549 100644 --- a/tools/build/wheel_win32_before_build.bat +++ b/tools/build/wheel_win32_before_build.bat @@ -1,26 +1,7 @@ @echo on -pip install delvewheel - -REM I really want to write "REM", like when I had a C=64 -REM (I am joking of course: I never wrote a comment when I had a C=64) -REM Broken since 2023-05-21, Failing with the error: -REM -REM postgresql (exited 1) - postgresql not installed. An error occurred during -REM installation: Unable to resolve dependency 'postgresql15 (= 15.3)'. -REM -REM Weeks later the error changed: -REM -REM Unable to resolve dependency 'postgresql15': Unable to resolve -REM dependencies. REM 'postgresql15 15.0.1' is not compatible with -REM 'postgresql 15.3.0 constraint: postgresql15 (= 15.3.0)'. -REM -REM choco upgrade postgresql +pip install delvewheel -REM On https://community.chocolatey.org/packages/postgresql15/15.0.1#discussion -REM I found the following command in a comment: -choco install postgresql15 --version 15.0.1 -REM which I'm going to randomly try. +vcpkg install libpq:x64-windows-release -REM See https://community.chocolatey.org/packages/postgresql15#install -REM for the last package available (bump the version number in the url). +pipx install .\tools\build\pg_config_vcpkg_stub\