From: jonathan vanasco Date: Fri, 2 Dec 2022 17:37:30 +0000 (-0500) Subject: Returned Github Actions support for py27 and py36. X-Git-Tag: rel_1_4_45~13 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3f1e6303f0f53cd6239b2a6227c8cd55789a175f;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Returned Github Actions support for py27 and py36. GitHub recently upgraded the `ubuntu-latest` label from `ubuntu-20.04` to `ubuntu-22.04`. The `ubuntu-22.04` image removed support for py27 and py36. To return support, the affected jobs have been duplicated to `-legacy` versions. The -legacy versions of jobs run py27 and py36 on a pinned `ubuntu-20.04` os. The existing jobs continue to run py37+ on ubuntu-latest, as that platform may continue to benefit from patch releases on the python versions. Change-Id: I0f063723cb993fab89bc64c89df6dfcaf4dbe5a5 --- diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yaml index 8353c0ec94..cf10ab88aa 100644 --- a/.github/workflows/create-wheels.yaml +++ b/.github/workflows/create-wheels.yaml @@ -123,6 +123,7 @@ jobs: twine upload --skip-existing dist/* make-wheel-linux: + # any changes should be duplicated in `make-wheel-linux-legacy` name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -131,9 +132,6 @@ jobs: - "ubuntu-latest" python-version: # the versions are - as specified in PEP 425. - - cp27-cp27m - - cp27-cp27mu - - cp36-cp36m - cp37-cp37m - cp38-cp38 - cp39-cp39 @@ -280,7 +278,164 @@ jobs: pip install -U twine twine upload --skip-existing dist/*manylinux* + make-wheel-linux-legacy: + # this is identical to `make-wheel-linux`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - "ubuntu-20.04" + python-version: + # the versions are - as specified in PEP 425. + - cp27-cp27m + - cp27-cp27mu + - cp36-cp36m + architecture: + - x64 + + include: + - python-version: "cp27-cp27m" + extra-requires: "mock" + - python-version: "cp27-cp27mu" + extra-requires: "mock" + + fail-fast: false + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Get python version + id: linux-py-version + env: + py_tag: ${{ matrix.python-version }} + # the command `echo "::set-output ...` is used to create an step output that can be used in following steps + # this is from https://github.community/t5/GitHub-Actions/Using-the-output-of-run-inside-of-if-condition/td-p/33920 + run: | + version="`echo $py_tag | sed --regexp-extended 's/cp([0-9])([0-9]+)-.*/\1.\2/g'`" + version=$([[ $version = "3.11" ]] && echo 3.11.0-rc - 3.11 || echo $version ) + echo $version + echo "::set-output name=python-version::$version" + + - name: Remove tag_build from setup.cfg + # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel + # otherwise it gets tagged with `dev0` + shell: pwsh + # This is equivalent to the sed commands: + # `sed -i '/tag_build=dev/d' setup.cfg` + # `sed -i '/tag_build = dev/d' setup.cfg` + + # `-replace` uses a regexp match + # alternative form: `(get-content setup.cfg) | foreach-object{$_ -replace "tag_build.=.dev",""} | set-content setup.cfg` + run: | + (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg + + - name: Create wheel for manylinux1 and manylinux2010 for py3 + if: ${{ matrix.python-version != 'cp27-cp27m' && matrix.python-version != 'cp27-cp27mu' && matrix.python-version != 'cp311-cp311' }} + # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux + # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel + # change the tag of this image to change the image used + uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2010_x86_64 + # this action generates 3 wheels in dist/. linux, manylinux1 and manylinux2010 + with: + # python-versions is the output of the previous step and is in the form -. Eg cp27-cp27mu + python-versions: ${{ matrix.python-version }} + build-requirements: "setuptools>=44 wheel>=0.34" + # Create the wheel using --no-use-pep517 since locally we have pyproject + # This flag should be removed once sqlalchemy supports pep517 + # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies + pip-wheel-args: "-w ./dist --no-use-pep517 -v --no-deps" + + - name: Create wheel for manylinux2014 for py3 + if: ${{ matrix.python-version != 'cp27-cp27m' && matrix.python-version != 'cp27-cp27mu' }} + # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux + # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel + # change the tag of this image to change the image used + uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2014_x86_64 + # this action generates 2 wheels in dist/. linux and manylinux2014 + with: + # python-versions is the output of the previous step and is in the form -. Eg cp27-cp27mu + python-versions: ${{ matrix.python-version }} + build-requirements: "setuptools>=44 wheel>=0.34" + # Create the wheel using --no-use-pep517 since locally we have pyproject + # This flag should be removed once sqlalchemy supports pep517 + # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies + pip-wheel-args: "-w ./dist --no-use-pep517 -v --no-deps" + + - name: Create wheel for manylinux py2 + if: ${{ matrix.python-version == 'cp27-cp27m' || matrix.python-version == 'cp27-cp27mu' }} + # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux + # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel + # change the tag of this image to change the image used + uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux1_x86_64 + # this action generates 2 wheels in dist/. linux and manylinux1 + with: + # python-versions is the output of the previous step and is in the form -. Eg cp27-cp27mu + python-versions: ${{ matrix.python-version }} + build-requirements: "setuptools>=44 wheel>=0.34" + # Create the wheel using --no-use-pep517 since locally we have pyproject + # This flag should be removed once sqlalchemy supports pep517 + # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies + pip-wheel-args: "-w ./dist --no-use-pep517 -v --no-deps" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ steps.linux-py-version.outputs.python-version }} + architecture: ${{ matrix.architecture }} + + - name: Check created wheel + # check that the wheel is compatible with the current installation. + # If it is then does: + # - install the created wheel without using the pypi index + # - check the c extension + # - runs the tests + run: | + pip install 'packaging>=20.4' + if python .github/workflows/scripts/can_install.py "${{ matrix.python-version }}" + then + pip install greenlet "importlib-metadata;python_version<'3.8'" + pip install -f dist --no-index sqlalchemy + python -c 'from sqlalchemy.util import has_compiled_ext; assert has_compiled_ext()' + pip install pytest pytest-xdist ${{ matrix.extra-requires }} + pytest -n2 -q test --nomemory --notimingintensive + else + echo Not compatible. Skipping install. + fi + + - name: Upload wheels to release + # upload the generated wheels to the github release + uses: sqlalchemyorg/upload-release-assets@sa + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + files: 'dist/*manylinux*' + + - name: Set up Python for twine + # twine on py2 is very old and is no longer updated, so we change to python 3.8 before upload + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Publish wheel + # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify + # additional options + # We upload both manylinux1 and manylinux2010 wheels. pip will download the appropriate one according to the system. + # manylinux1 is an older format and is now not very used since many environments can use manylinux2010 + # currently (April 2020) manylinux2014 is still wip, so we do not generate it. + env: + TWINE_USERNAME: __token__ + # replace TWINE_PASSWORD with token for real pypi + # TWINE_PASSWORD: ${{ secrets.test_pypi_token }} + TWINE_PASSWORD: ${{ secrets.pypi_token }} + run: | + pip install -U twine + twine upload --skip-existing dist/*manylinux* + + make-wheel-linux-arm64: + # any changes should be duplicated in `make-wheel-linux-arm64-legacy` name: ${{ matrix.python-version }}-arm64-${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -289,7 +444,6 @@ jobs: - "ubuntu-latest" python-version: # the versions are - as specified in PEP 425. - - cp36-cp36m - cp37-cp37m - cp38-cp38 - cp39-cp39 @@ -375,3 +529,96 @@ jobs: run: | pip install -U twine twine upload --skip-existing dist/*manylinux* + + make-wheel-linux-arm64-legacy: + # this is identical to `make-wheel-linux-arm64`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: ${{ matrix.python-version }}-arm64-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - "ubuntu-20.04" + python-version: + # the versions are - as specified in PEP 425. + - cp36-cp36m + + fail-fast: false + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Remove tag_build from setup.cfg + # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel + # otherwise it gets tagged with `dev0` + shell: pwsh + # This is equivalent to the sed commands: + # `sed -i '/tag_build=dev/d' setup.cfg` + # `sed -i '/tag_build = dev/d' setup.cfg` + + # `-replace` uses a regexp match + # alternative form: `(get-content setup.cfg) | foreach-object{$_ -replace "tag_build.=.dev",""} | set-content setup.cfg` + run: | + (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg + + - name: Set up emulation + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Create wheel for manylinux2014 + # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux + # the action uses the image for manylinux2014 but can generate also a manylinux1 wheel + # change the tag of this image to change the image used + uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2014_aarch64 + # this action generates 2 wheels in dist/. linux and manylinux2014 + with: + # python-versions is the output of the previous step and is in the form -. Eg cp37-cp37mu + python-versions: ${{ matrix.python-version }} + build-requirements: "setuptools>=44 wheel>=0.34" + # Create the wheel using --no-use-pep517 since locally we have pyproject + # This flag should be removed once sqlalchemy supports pep517 + # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies + pip-wheel-args: "-w ./dist --no-use-pep517 -v --no-deps" + + - name: Check created wheel + # check that the wheel is compatible with the current installation. + # - runs the tests + uses: docker://quay.io/pypa/manylinux2014_aarch64 + with: + args: | + bash -c " + export PATH=/opt/python/${{ matrix.python-version }}/bin:$PATH && + python --version && + pip install greenlet \"importlib-metadata;python_version<'3.8'\" && + pip install -f dist --no-index sqlalchemy && + python -c 'from sqlalchemy.util import has_compiled_ext; assert has_compiled_ext()' && + pip install pytest pytest-xdist ${{ matrix.extra-requires }} && + pytest -n2 -q test --nomemory --notimingintensive" + + - name: Upload wheels to release + # upload the generated wheels to the github release + uses: sqlalchemyorg/upload-release-assets@sa + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + files: 'dist/*manylinux*' + + - name: Set up Python for twine + # Setup python after creating the wheel, otherwise LD_LIBRARY_PATH gets set and it will break wheel generation + # twine on py2 is very old and is no longer updated, so we change to python 3.8 before upload + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Publish wheel + # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify + # additional options + # We upload manylinux2014 arm64 wheels. pip will download the appropriate one according to the system. + env: + TWINE_USERNAME: __token__ + # replace TWINE_PASSWORD with token for real pypi + # TWINE_PASSWORD: ${{ secrets.test_pypi_token }} + TWINE_PASSWORD: ${{ secrets.pypi_token }} + run: | + pip install -U twine + twine upload --skip-existing dist/*manylinux* \ No newline at end of file diff --git a/.github/workflows/run-on-pr.yaml b/.github/workflows/run-on-pr.yaml index 087f1bc332..2f11b7dad8 100644 --- a/.github/workflows/run-on-pr.yaml +++ b/.github/workflows/run-on-pr.yaml @@ -17,6 +17,7 @@ permissions: jobs: run-test-amd64: + # any changes to this job should be duplicated in `run-mypy-legacy` name: ${{ matrix.python-version }}-${{ matrix.build-type }}-${{ matrix.architecture }}-${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -25,7 +26,6 @@ jobs: os: - "ubuntu-latest" python-version: - - "2.7" - "3.10" build-type: - "cext" @@ -55,6 +55,46 @@ jobs: - name: Run tests run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} + run-test-amd64-legacy: + # this is identical to `run-test-amd64`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: ${{ matrix.python-version }}-${{ matrix.build-type }}-${{ matrix.architecture }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + # run this job using this matrix, excluding some combinations below. + matrix: + os: + - "ubuntu-20.04" + python-version: + - "2.7" + build-type: + - "cext" + - "nocext" + architecture: + - x64 + # abort all jobs as soon as one fails + fail-fast: true + + # steps to run in each job. Some are github actions, others run shell commands + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox setuptools + pip list + + - name: Run tests + run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} + run-mypy: name: mypy-${{ matrix.python-version }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/run-test.yaml b/.github/workflows/run-test.yaml index 1c97f64bc1..eb7aafae9f 100644 --- a/.github/workflows/run-test.yaml +++ b/.github/workflows/run-test.yaml @@ -21,6 +21,7 @@ permissions: jobs: run-test: + # any changes to this job should be duplicated in `run-test-legacy` name: ${{ matrix.python-version }}-${{ matrix.build-type }}-${{ matrix.architecture }}-${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: @@ -31,8 +32,6 @@ jobs: - "windows-latest" - "macos-latest" python-version: - - "2.7" - - "3.6" - "3.7" - "3.8" - "3.9" @@ -53,7 +52,76 @@ jobs: # - python-version: "pypy-3.9" # pytest-args: "-k 'not test_autocommit_on and not test_turn_autocommit_off_via_default_iso_level and not test_autocommit_isolation_level'" # add aiosqlite on linux - - os: "ubuntu-latest" + - os: "ubuntu-20.04" + pytest-args: "--dbdriver pysqlite --dbdriver aiosqlite" + + exclude: + # c-extensions fail to build on windows for python 2.7 + - os: "windows-latest" + python-version: "2.7" + build-type: "cext" + # linux and osx do not have x86 python + - os: "ubuntu-20.04" + architecture: x86 + - os: "macos-latest" + architecture: x86 + # pypy does not have cext + # - python-version: "pypy-3.9" + # build-type: "cext" + # - os: "windows-latest" + # python-version: "pypy-3.9" + # architecture: x86 + + fail-fast: false + + # steps to run in each job. Some are github actions, others run shell commands + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox setuptools + pip list + + - name: Run tests + run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} + + run-test-legacy: + # this is identical to `run-test`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: ${{ matrix.python-version }}-${{ matrix.build-type }}-${{ matrix.architecture }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + # run this job using this matrix, excluding some combinations below. + matrix: + os: + - "ubuntu-20.04" + - "windows-latest" + - "macos-latest" + python-version: + - "2.7" + - "3.6" + build-type: + - "cext" + - "nocext" + architecture: + - x64 + - x86 + + include: + # autocommit tests fail on the ci for some reason + # - python-version: "pypy-3.9" + # pytest-args: "-k 'not test_autocommit_on and not test_turn_autocommit_off_via_default_iso_level and not test_autocommit_isolation_level'" + # add aiosqlite on linux + - os: "ubuntu-20.04" pytest-args: "--dbdriver pysqlite --dbdriver aiosqlite" exclude: @@ -62,7 +130,7 @@ jobs: python-version: "2.7" build-type: "cext" # linux and osx do not have x86 python - - os: "ubuntu-latest" + - os: "ubuntu-20.04" architecture: x86 - os: "macos-latest" architecture: x86 @@ -96,12 +164,12 @@ jobs: run: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} run-test-arm64: + # any changes to this job should be duplicated in `run-test-arm64` name: ${{ matrix.python-version }}-${{ matrix.build-type }}-arm64-ubuntu-latest runs-on: ubuntu-latest strategy: matrix: python-version: - - cp36-cp36m - cp37-cp37m - cp38-cp38 - cp39-cp39 @@ -134,7 +202,44 @@ jobs: tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} " + run-test-arm64-legacy: + # this is identical to `run-test-arm64-legacy`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: ${{ matrix.python-version }}-${{ matrix.build-type }}-arm64-ubuntu-20.04 + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: + - cp36-cp36m + build-type: + - "cext" + - "nocext" + + fail-fast: false + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up emulation + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Run tests + uses: docker://quay.io/pypa/manylinux2014_aarch64 + with: + args: | + bash -c " + export PATH=/opt/python/${{ matrix.python-version }}/bin:$PATH && + python --version && + python -m pip install --upgrade pip && + pip install --upgrade tox setuptools && + pip list && + tox -e github-${{ matrix.build-type }} -- -q --nomemory --notimingintensive ${{ matrix.pytest-args }} + " + run-mypy: + # any changes to this job should be duplicated in `run-mypy-legacy` name: mypy-${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: @@ -143,7 +248,6 @@ jobs: os: - "ubuntu-latest" python-version: - - "3.6" - "3.7" - "3.8" - "3.9" @@ -171,6 +275,40 @@ jobs: - name: Run tests run: tox -e mypy ${{ matrix.pytest-args }} + run-mypy-legacy: + # this is identical to `run-mypy`, but pins ubuntu to 20.04 + # ubuntu-20.04 is necessary to run: py27, py36 + name: mypy-${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + # run this job using this matrix, excluding some combinations below. + matrix: + os: + - "ubuntu-20.04" + python-version: + - "3.6" + fail-fast: false + + # steps to run in each job. Some are github actions, others run shell commands + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade tox setuptools + pip list + + - name: Run tests + run: tox -e mypy ${{ matrix.pytest-args }} + run-pep8: name: pep8-${{ matrix.python-version }} runs-on: ${{ matrix.os }}