Use cibuildwheel to create wheels

Using cibuildwheel the following wheels are created:
- windows x64 and x84
- macos x64 and arm
- linux x64 and arm on manylinux and mosulinux (for alpine)
- create a pure python wheel (for pypy and other archs)

Fixes: #6702
Fixes: #7607
Closes: #7992
Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/7992
Pull-request-sha: 61d5e24e5b76c97db73aa2507af7f5c2d3a948fa

Change-Id: If0c0b353766e0b61d421789d619eb2b940c08ad0

diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yaml
index 56505522663a5d163ac09042e00888f1df6402d3..49ad3f91cf2ad5986836615e191078e357971233 100644 (file)
--- a/.github/workflows/create-wheels.yaml
+++ b/.github/workflows/create-wheels.yaml
@@ -6,46 +6,35 @@ on:
     types: [created]
 
 env:
-  # set this so the sqlalchemy test uses the installed version and not the local one
-  PYTHONNOUSERSITE: 1
   # comment TWINE_REPOSITORY_URL to use the real pypi. NOTE: change also the secret used in TWINE_PASSWORD
   # TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
 
 jobs:
-  # two jobs are defined make-wheel-win-osx and make-wheel-linux.
-  # they do the the same steps, but linux wheels need to be build to target manylinux
-  make-wheel-win-osx:
-    name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }}
+  build_wheels:
+    name: Build ${{ matrix.wheel_mode }} wheels on ${{ matrix.os }} ${{ matrix.linux_archs }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os:
-          - "windows-latest"
-          - "macos-latest"
-        python-version:
-          - "3.7"
-          - "3.8"
-          - "3.9"
-          - "3.10"
-        architecture:
-          - x64
-          - x86
-
-        exclude:
-          - os: "macos-latest"
-            architecture: x86
+        include:
+          - os: windows-2022
+            wheel_mode: compiled
+          - os: macos-12
+            wheel_mode: compiled
+          # emulated wheels on linux take too much time, so run two jobs
+          - os: ubuntu-22.04
+            wheel_mode: compiled
+            linux_archs: "x86_64"
+          - os: ubuntu-22.04
+            wheel_mode: compiled
+            linux_archs: "aarch64"
+          # create pure python build
+          - os: ubuntu-22.04
+            wheel_mode: pure-python
 
       fail-fast: false
 
     steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: ${{ matrix.architecture }}
+      - uses: actions/checkout@v3
 
       - name: Remove tag_build from setup.cfg
         # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel
@@ -60,252 +49,48 @@ jobs:
         run: |
           (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg
 
-      - name: Create wheel
-        # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip --version
-          pip install 'setuptools>=44' 'wheel>=0.34'
-          pip list
-          pip wheel -w dist -v --no-deps .
-
-      - name: Install wheel
-        # install the created wheel without using the pypi index
-        run: |
-          pip install greenlet "importlib-metadata;python_version<'3.8'" &&
-          pip install -f dist --no-index sqlalchemy
-
-      - name: Check c extensions
-        run: |
-          python -c 'from sqlalchemy.util import has_compiled_ext; assert has_compiled_ext()'
-
-      - name: Test created wheel
-        # the mock reconnect test seems to fail on the ci in windows
-        run: |
-          pip install pytest pytest-xdist ${{ matrix.extra-requires }}
-          pytest -n2 -q test --nomemory --notimingintensive
-
-      - name: Upload wheels to release
-        # upload the generated wheels to the github release
-        uses: sqlalchemyorg/upload-release-assets@sa
+      # See details at https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation
+      - name: Set up QEMU on linux
+        if: ${{ runner.os == 'Linux' }}
+        uses: docker/setup-qemu-action@v2
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          files: 'dist/*.whl'
-
-      - name: Publish wheel
-        # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify
-        # additional options
-        env:
-          TWINE_USERNAME: __token__
-          # replace TWINE_PASSWORD with token for real pypi
-          # TWINE_PASSWORD: ${{ secrets.test_pypi_token }}
-          TWINE_PASSWORD: ${{ secrets.pypi_token }}
-        run: |
-          pip install -U twine
-          twine upload --skip-existing dist/*
+          platforms: all
 
-  make-wheel-linux:
-    name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os:
-          - "ubuntu-latest"
-        python-version:
-          # the versions are <python tag>-<abi tag> as specified in PEP 425.
-          - cp37-cp37m
-          - cp38-cp38
-          - cp39-cp39
-          - cp310-cp310
-        architecture:
-          - x64
-
-      fail-fast: false
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Get python version
-        id: linux-py-version
+      - name: Build compiled wheels
+        if: ${{ matrix.wheel_mode == 'compiled' }}
+        uses: pypa/cibuildwheel@v2.9.0
         env:
-          py_tag: ${{ matrix.python-version }}
-        # the command `echo "::set-output ...` is used to create an step output that can be used in following steps
-        # this is from https://github.community/t5/GitHub-Actions/Using-the-output-of-run-inside-of-if-condition/td-p/33920
-        run: |
-          version="`echo $py_tag | sed --regexp-extended 's/cp([0-9])([0-9]+)-.*/\1.\2/g'`"
-          echo $version
-          echo "::set-output name=python-version::$version"
-
-      - name: Remove tag_build from setup.cfg
-        # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel
-        # otherwise it gets tagged with `dev0`
-        shell: pwsh
-        # This is equivalent to the sed commands:
-        # `sed -i '/tag_build=dev/d' setup.cfg`
-        # `sed -i '/tag_build = dev/d' setup.cfg`
-
-        # `-replace` uses a regexp match
-        # alternative form: `(get-content setup.cfg) | foreach-object{$_ -replace "tag_build.=.dev",""} | set-content setup.cfg`
-        run: |
-          (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg
+          CIBW_ARCHS_LINUX: ${{ matrix.linux_archs }}
 
-      - name: Create wheel for manylinux1 and manylinux2010
-        # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux
-        # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel
-        # change the tag of this image to change the image used
-        uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2010_x86_64
-        # this action generates 3 wheels in dist/. linux, manylinux1 and manylinux2010
-        with:
-          # python-versions is the output of the previous step and is in the form <python tag>-<abi tag>. Eg cp27-cp27mu
-          python-versions: ${{ matrix.python-version }}
-          build-requirements: "setuptools>=47 wheel>=0.34 cython>=0.29.24"
-          # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies
-          pip-wheel-args: "-w ./dist -v --no-deps"
-
-      - name: Create wheel for manylinux2014
-        # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux
-        # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel
-        # change the tag of this image to change the image used
-        uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_x86_64
-        # this action generates 2 wheels in dist/. linux and manylinux2014
-        with:
-          # python-versions is the output of the previous step and is in the form <python tag>-<abi tag>. Eg cp27-cp27mu
-          python-versions: ${{ matrix.python-version }}
-          build-requirements: "setuptools>=47 wheel>=0.34 cython>=0.29.24"
-          # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies
-          pip-wheel-args: "-w ./dist -v --no-deps"
 
-      - name: Set up Python
+      - name: Set up Python for twine and pure-python wheel
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ steps.linux-py-version.outputs.python-version }}
-          architecture: ${{ matrix.architecture }}
-
-      - name: Check created wheel
-        # check that the wheel is compatible with the current installation.
-        # If it is then does:
-        # - install the created wheel without using the pypi index
-        # - check the c extension
-        # - runs the tests
-        run: |
-          pip install 'packaging>=20.4'
-          if python .github/workflows/scripts/can_install.py "${{ matrix.python-version }}"
-          then
-            pip install greenlet "importlib-metadata;python_version<'3.8'"
-            pip install -f dist --no-index sqlalchemy
-            python -c 'from sqlalchemy.util import has_compiled_ext; assert has_compiled_ext()'
-            pip install pytest pytest-xdist ${{ matrix.extra-requires }}
-            pytest -n2 -q test --nomemory --notimingintensive
-          else
-            echo Not compatible. Skipping install.
-          fi
-
-      - name: Upload wheels to release
-        # upload the generated wheels to the github release
-        uses: sqlalchemyorg/upload-release-assets@sa
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          files: 'dist/*manylinux*'
-
-      - name: Publish wheel
-        # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify
-        # additional options
-        # We upload both manylinux1 and manylinux2010 wheels. pip will download the appropriate one according to the system.
-        # manylinux1 is an older format and is now not very used since many environments can use manylinux2010
-        # currently (April 2020) manylinux2014 is still wip, so we do not generate it.
-        env:
-          TWINE_USERNAME: __token__
-          # replace TWINE_PASSWORD with token for real pypi
-          # TWINE_PASSWORD: ${{ secrets.test_pypi_token }}
-          TWINE_PASSWORD: ${{ secrets.pypi_token }}
-        run: |
-          pip install -U twine
-          twine upload --skip-existing dist/*manylinux*
-
-  make-wheel-linux-arm64:
-    name: ${{ matrix.python-version }}-arm64-${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os:
-          - "ubuntu-latest"
-        python-version:
-          # the versions are <python tag>-<abi tag> as specified in PEP 425.
-          - cp37-cp37m
-          - cp38-cp38
-          - cp39-cp39
-          - cp310-cp310
+          python-version: "3.10"
 
-      fail-fast: false
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Remove tag_build from setup.cfg
-        # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel
-        # otherwise it gets tagged with `dev0`
-        shell: pwsh
-        # This is equivalent to the sed commands:
-        # `sed -i '/tag_build=dev/d' setup.cfg`
-        # `sed -i '/tag_build = dev/d' setup.cfg`
-
-        # `-replace` uses a regexp match
-        # alternative form: `(get-content setup.cfg) | foreach-object{$_ -replace "tag_build.=.dev",""} | set-content setup.cfg`
-        run: |
-          (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg
-
-      - name: Set up emulation
+      - name: Build pure-python wheel
+        if: ${{ matrix.wheel_mode == 'pure-python' && runner.os == 'Linux' }}
         run: |
-          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-      - name: Create wheel for manylinux2014
-        # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux
-        # the action uses the image for manylinux2014 but can generate also a manylinux1 wheel
-        # change the tag of this image to change the image used
-        uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_aarch64
-        # this action generates 2 wheels in dist/. linux and manylinux2014
-        with:
-          # python-versions is the output of the previous step and is in the form <python tag>-<abi tag>. Eg cp37-cp37mu
-          python-versions: ${{ matrix.python-version }}
-          build-requirements: "setuptools>=47 wheel>=0.34 cython>=0.29.24"
-          # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies
-          pip-wheel-args: "-w ./dist -v --no-deps"
+          python -m pip install --upgrade pip
+          pip --version
+          pip install build
+          pip list
+          DISABLE_SQLALCHEMY_CEXT=y python -m build --wheel --outdir ./wheelhouse
 
-      - name: Check created wheel
-        # check that the wheel is compatible with the current installation.
-        # - runs the tests
-        uses: docker://quay.io/pypa/manylinux2014_aarch64
-        with:
-          args: |
-            bash -c "
-            export PATH=/opt/python/${{ matrix.python-version }}/bin:$PATH &&
-            python --version &&
-            pip install greenlet \"importlib-metadata;python_version<'3.8'\" &&
-            pip install -f dist --no-index sqlalchemy &&
-            python -c 'from sqlalchemy.util import has_compiled_ext; assert has_compiled_ext()' &&
-            pip install pytest pytest-xdist ${{ matrix.extra-requires }} &&
-            pytest -n2 -q test --nomemory --notimingintensive"
+      # - uses: actions/upload-artifact@v3
+      #   with:
+      #     path: ./wheelhouse/*.whl
 
       - name: Upload wheels to release
         # upload the generated wheels to the github release
         uses: sqlalchemyorg/upload-release-assets@sa
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          files: 'dist/*manylinux*'
-
-      - name: Set up Python for twine
-        # Setup python after creating the wheel, otherwise LD_LIBRARY_PATH gets set and it will break wheel generation
-        # twine on py2 is very old and is no longer updated, so we change to python 3.8 before upload
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
+          files: './wheelhouse/*.whl'
 
       - name: Publish wheel
         # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify
         # additional options
-        # We upload manylinux2014 arm64 wheels. pip will download the appropriate one according to the system.
         env:
           TWINE_USERNAME: __token__
           # replace TWINE_PASSWORD with token for real pypi
@@ -313,4 +98,4 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.pypi_token }}
         run: |
           pip install -U twine
-          twine upload --skip-existing dist/*manylinux*
+          twine upload --skip-existing ./wheelhouse/*

diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py
index 8e51139544e7843ba29a482276a0a48738c50bb2..494e7d5ab5b9b20f469897d48b24228fe9464508 100644 (file)
--- a/lib/sqlalchemy/testing/plugin/plugin_base.py
+++ b/lib/sqlalchemy/testing/plugin/plugin_base.py
@@ -135,6 +135,13 @@ def setup_options(make_option):
         "this is now equivalent to the pytest -m 'not timing_intensive' "
         "mark expression",
     )
+    make_option(
+        "--nomypy",
+        action="callback",
+        zeroarg_callback=_set_tag_exclude("mypy"),
+        help="Don't run mypy typing tests; "
+        "this is now equivalent to the pytest -m 'not mypy' mark expression",
+    )
     make_option(
         "--profile-sort",
         type=str,
@@ -260,10 +267,12 @@ def restore_important_follower_config(dict_):
     """
 
 
-def read_config():
+def read_config(root_path):
     global file_config
     file_config = configparser.ConfigParser()
-    file_config.read(["setup.cfg", "test.cfg"])
+    file_config.read(
+        [str(root_path / "setup.cfg"), str(root_path / "test.cfg")]
+    )
 
 
 def pre_begin(opt):

diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py
index cea07b3053efabf5787c71bb4104a8f6369257c7..0a70f4008e345c7e14a7abe871ffd151f2760f60 100644 (file)
--- a/lib/sqlalchemy/testing/plugin/pytestplugin.py
+++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py
@@ -69,10 +69,10 @@ def pytest_addoption(parser):
         group.addoption(name, **kw)
 
     plugin_base.setup_options(make_option)
-    plugin_base.read_config()
 
 
-def pytest_configure(config):
+def pytest_configure(config: pytest.Config):
+    plugin_base.read_config(config.rootpath)
     if plugin_base.exclude_tags or plugin_base.include_tags:
         new_expr = " and ".join(
             list(plugin_base.include_tags)

diff --git a/pyproject.toml b/pyproject.toml
index 3807b9c103a05b7d8debac530e4bc737291c39ac..bba572d5090f84b1429b4c590257e3480ac1574b 100644 (file)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -61,3 +61,22 @@ warn_unused_ignores = false
 strict = true
 
 
+
+[tool.cibuildwheel]
+test-requires = "pytest pytest-xdist"
+test-command = "pytest -c {project}/pyproject.toml -n2 -q --nomemory --notimingintensive --nomypy {project}/test"
+
+build = "*"
+# python 3.6 is no longer supported by sqlalchemy
+# pypy uses the universal wheel fallback, since it does not use any compiled extension
+skip = "cp36-* pp*"
+# TODO: remove this skip once action support arm macs
+test-skip = "*-macosx_arm64"
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "arm64"]
+
+# On an Linux Intel runner with qemu installed, build Intel and ARM wheels
+# NOTE: this is overriden in the pipeline using the CIBW_ARCHS_LINUX env variable to speed up the build
+[tool.cibuildwheel.linux]
+archs = ["x86_64", "aarch64"]

diff --git a/setup.py b/setup.py
index 60495892258b5394de8f052a32e81a022077adb6..c60a214141f7947ebd50ac6dca79348e80a73bb7 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,6 @@ import platform
 import sys
 
 from setuptools import __version__
-from setuptools import Distribution as _Distribution
 from setuptools import setup
 
 if not int(__version__.partition(".")[0]) >= 47:
@@ -96,17 +95,6 @@ else:
     ext_modules = []
 
 
-class Distribution(_Distribution):
-    def has_ext_modules(self):
-        # We want to always claim that we have ext_modules. This will be fine
-        # if we don't actually have them (such as on PyPy) because nothing
-        # will get built, however we don't want to provide an overally broad
-        # Wheel package when building a wheel without C support. This will
-        # ensure that Wheel knows to treat us as if the build output is
-        # platform specific.
-        return True
-
-
 def status_msgs(*msgs):
     print("*" * 75)
     for msg in msgs:
@@ -127,7 +115,7 @@ def run_setup(with_cext):
 
         kwargs["ext_modules"] = []
 
-    setup(cmdclass=cmdclass, distclass=Distribution, **kwargs)
+    setup(cmdclass=cmdclass, **kwargs)
 
 
 if not cpython: