From: Viktor Szakats Date: Wed, 24 Sep 2025 15:22:52 +0000 (+0200) Subject: GHA: use pip `requirements.txt` with pins, and more venv X-Git-Tag: rc-8_17_0-3~460 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7d5f8be532c19ec73063aaa4f27057047bdae5ac;p=thirdparty%2Fcurl.git GHA: use pip `requirements.txt` with pins, and more venv - requirements.txt: shorten copyright headers. - requirements.txt: pin packages to versions. - GHA/windows: use `tests/requirements.txt`. Pick a `cryptography` package version that satifies both `impacket` and pytests dependencies. - GHA/checksrc: move pip deps into a new `requirements.txt`. To make Dependabot detect and bump them. - GHA/checksrc: replace apt packages for python test deps with pip install `tests/**/requirements.txt` to a venv. - GHA/checksrc: use venv and drop `--break-system-packages`. - GHA/linux: fix to actually activate venvs. Follow-up to 2638570241cb9e68240d7621f0213916334a4765 #15578 - GHA/linux: fixup (did not cause an issue) Follow-up to d75785c7dea214d12525beb659694d3fcc483731 #18660 - GHA: create venvs later, simplify commands. - GHA: sync pip command-line options, e.g. drop progress-bar, everywhere. Assisted-by: Dan Fandrich Closes #18708 --- diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt new file mode 100644 index 0000000000..f94a9d93a8 --- /dev/null +++ b/.github/scripts/requirements.txt @@ -0,0 +1,8 @@ +# Copyright (C) Daniel Stenberg, , et al. +# +# SPDX-License-Identifier: curl + +cmakelang == 0.6.13 +codespell == 2.4.1 +pytype == 2024.10.11 +ruff == 0.11.9 diff --git a/.github/workflows/checksrc.yml b/.github/workflows/checksrc.yml index a0ff120bec..5aca2f941a 100644 --- a/.github/workflows/checksrc.yml +++ b/.github/workflows/checksrc.yml @@ -55,18 +55,15 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo apt-get -o Dpkg::Use-Pty=0 update - sudo rm -f /var/lib/man-db/auto-update - sudo apt-get -o Dpkg::Use-Pty=0 install \ - python3-pip python3-networkx python3-pydot python3-yaml \ - python3-toml python3-markupsafe python3-jinja2 python3-tabulate \ - python3-typing-extensions python3-libcst python3-impacket \ - python3-websockets python3-pytest python3-filelock python3-pytest-xdist - python3 -m pip install --break-system-packages cmakelang==0.6.13 pytype==2024.10.11 ruff==0.11.9 codespell==2.4.1 + python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary \ + -r .github/scripts/requirements.txt \ + -r tests/http/requirements.txt \ + -r tests/requirements.txt - name: 'codespell' run: | + source ~/venv/bin/activate codespell --version .github/scripts/codespell.sh @@ -78,13 +75,19 @@ jobs: .github/scripts/typos.sh - name: 'cmakelint' - run: scripts/cmakelint.sh + run: | + source ~/venv/bin/activate + scripts/cmakelint.sh - name: 'pytype' - run: find . -name '*.py' -exec pytype -j auto -k {} + + run: | + source ~/venv/bin/activate + find . -name '*.py' -exec pytype -j auto -k {} + - name: 'ruff' - run: scripts/pythonlint.sh + run: | + source ~/venv/bin/activate + scripts/pythonlint.sh reuse: name: 'REUSE' diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml index 61164913ad..019790b68e 100644 --- a/.github/workflows/http3-linux.yml +++ b/.github/workflows/http3-linux.yml @@ -512,7 +512,6 @@ jobs: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libidn2-0-dev libldap-dev libuv1-dev \ ${INSTALL_PACKAGES} \ ${MATRIX_INSTALL_PACKAGES} - python3 -m venv ~/venv echo 'CC=gcc-12' >> "$GITHUB_ENV" echo 'CXX=g++-12' >> "$GITHUB_ENV" @@ -726,8 +725,8 @@ jobs: - name: 'install test prereqs' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} run: | - source ~/venv/bin/activate - python3 -m pip install -r tests/requirements.txt + python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/requirements.txt - name: 'run tests' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} @@ -744,8 +743,8 @@ jobs: - name: 'install pytest prereqs' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} run: | - source ~/venv/bin/activate - python3 -m pip install -r tests/http/requirements.txt + [ -d ~/venv ] || python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/http/requirements.txt - name: 'run pytest event based' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4f16438267..556ff8df1e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -321,8 +321,9 @@ jobs: libpsl-dev zlib1g-dev libbrotli-dev libzstd-dev \ ${INSTALL_PACKAGES} \ ${MATRIX_INSTALL_PACKAGES} - [ -n "${INSTALL_PACKAGES_BREW}" ] && /home/linuxbrew/.linuxbrew/bin/brew install ${INSTALL_PACKAGES_BREW} - python3 -m venv ~/venv + if [ -n "${INSTALL_PACKAGES_BREW}" ]; then + /home/linuxbrew/.linuxbrew/bin/brew install ${INSTALL_PACKAGES_BREW} + fi - name: 'install prereqs' if: ${{ contains(matrix.build.name, 'i686') }} @@ -335,7 +336,6 @@ jobs: libtool autoconf automake pkgconf stunnel4 \ libpsl-dev:i386 libbrotli-dev:i386 libzstd-dev:i386 \ ${MATRIX_INSTALL_PACKAGES} - python3 -m venv ~/venv - name: 'install dependencies' if: ${{ startsWith(matrix.build.container, 'alpine') }} @@ -657,8 +657,8 @@ jobs: - name: 'install test prereqs' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') && matrix.build.container == null }} run: | - [ -x ~/venv/bin/activate ] && source ~/venv/bin/activate - python3 -m pip install -r tests/requirements.txt + python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/requirements.txt - name: 'run tests' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} @@ -675,7 +675,7 @@ jobs: fi fi fi - [ -x ~/venv/bin/activate ] && source ~/venv/bin/activate + [ -f ~/venv/bin/activate ] && source ~/venv/bin/activate if [[ "${MATRIX_INSTALL_STEPS}" = *'codeset-test'* ]]; then locale || true export LC_ALL=C @@ -691,8 +691,8 @@ jobs: - name: 'install pytest prereqs' if: ${{ contains(matrix.build.install_steps, 'pytest') }} run: | - [ -x ~/venv/bin/activate ] && source ~/venv/bin/activate - python3 -m pip install -r tests/http/requirements.txt + [ -d ~/venv ] || python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/http/requirements.txt - name: 'run pytest' if: ${{ contains(matrix.build.install_steps, 'pytest') }} @@ -700,7 +700,7 @@ jobs: PYTEST_ADDOPTS: '--color=yes' PYTEST_XDIST_AUTO_NUM_WORKERS: 4 run: | - [ -x ~/venv/bin/activate ] && source ~/venv/bin/activate + [ -f ~/venv/bin/activate ] && source ~/venv/bin/activate if [ "${MATRIX_BUILD}" = 'cmake' ]; then cmake --build bld --verbose --target curl-pytest-ci else diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c111eaf419..88b7ee8800 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -489,8 +489,7 @@ jobs: if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} run: | python3 -m venv ~/venv - source ~/venv/bin/activate - python3 -m pip install -r tests/requirements.txt + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/requirements.txt - name: 'run tests' if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }} @@ -517,8 +516,8 @@ jobs: - name: 'install pytest prereqs' if: ${{ contains(matrix.build.install_steps, 'pytest') }} run: | - source ~/venv/bin/activate - python3 -m pip install -r tests/http/requirements.txt + [ -d ~/venv ] || python3 -m venv ~/venv + ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/http/requirements.txt - name: 'run pytest' if: ${{ contains(matrix.build.install_steps, 'pytest') }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0f802b6902..e5d66ccc87 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -658,7 +658,7 @@ jobs: timeout-minutes: 5 run: | /c/ProgramData/chocolatey/choco.exe install --yes --no-progress --limit-output --timeout 180 --force stunnel || true - python3 -m pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary impacket + python3 -m pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/requirements.txt perl --version | tee "$GITHUB_WORKSPACE"/perlversion - name: 'cache perl packages' @@ -1051,7 +1051,7 @@ jobs: fi /c/ProgramData/chocolatey/choco.exe install --yes --no-progress --limit-output --timeout 180 --force stunnel || true if [ "${MATRIX_IMAGE}" != 'windows-11-arm' ]; then # save 30-60 seconds, to counteract the slower test run step - python3 -m pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary impacket + python3 -m pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary -r tests/requirements.txt fi perl --version | tee "$GITHUB_WORKSPACE"/perlversion diff --git a/tests/http/requirements.txt b/tests/http/requirements.txt index c879605608..8dddcd1e1c 100644 --- a/tests/http/requirements.txt +++ b/tests/http/requirements.txt @@ -1,31 +1,10 @@ -# -*- coding: utf-8 -*- -#*************************************************************************** -# _ _ ____ _ -# Project ___| | | | _ \| | -# / __| | | | |_) | | -# | (__| |_| | _ <| |___ -# \___|\___/|_| \_\_____| -# # Copyright (C) Daniel Stenberg, , et al. # -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at https://curl.se/docs/copyright.html. -# -# You may opt to use, copy, modify, merge, publish, distribute and/or sell -# copies of the Software, and permit persons to whom the Software is -# furnished to do so, under the terms of the COPYING file. -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY -# KIND, either express or implied. -# # SPDX-License-Identifier: curl -# -########################################################################### -# -pytest -cryptography -filelock -websockets -psutil -pytest-xdist + +cryptography == 42.0.8 +filelock == 3.19.1 +psutil == 7.1.0 +pytest == 8.4.2 +pytest-xdist == 3.8.0 +websockets == 15.0.1 diff --git a/tests/requirements.txt b/tests/requirements.txt index 706043ac39..dab4784c5f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,26 +1,5 @@ -# -*- coding: utf-8 -*- -#*************************************************************************** -# _ _ ____ _ -# Project ___| | | | _ \| | -# / __| | | | |_) | | -# | (__| |_| | _ <| |___ -# \___|\___/|_| \_\_____| -# # Copyright (C) Daniel Stenberg, , et al. # -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at https://curl.se/docs/copyright.html. -# -# You may opt to use, copy, modify, merge, publish, distribute and/or sell -# copies of the Software, and permit persons to whom the Software is -# furnished to do so, under the terms of the COPYING file. -# -# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY -# KIND, either express or implied. -# # SPDX-License-Identifier: curl -# -########################################################################### -# -impacket + +impacket == 0.12.0