]> git.ipfire.org Git - thirdparty/sqlalchemy/sqlalchemy.git/commitdiff
ARM64 Support to Github Actions For Testing And Development Of Wheels
authorodidev <odidev@puresoftware.com>
Sun, 23 Aug 2020 18:54:56 +0000 (14:54 -0400)
committerFederico Caselli <cfederico87@gmail.com>
Mon, 24 Aug 2020 22:06:17 +0000 (00:06 +0200)
**Added**:
1. ARM64 jobs in Github Workflows For Testing And Development Of Wheels
2. Two script files for creating wheels and testing.

Resolves #5436

Signed-off-by: odidev <odidev@puresoftware.com>
Co-authored-by: Federico Caselli <cfederico87@gmail.com>
Closes: #5480
Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/5480
Pull-request-sha: 131ef520bbf863ad9b65ca3bf3c2d2d214405702

Change-Id: Ia10dadd93365ba4beeefbd27c060bc077c2d7aeb

.github/workflows/create-wheels.yaml
.github/workflows/run-on-pr.yaml
.github/workflows/run-test.yaml

index f1c8b04f1e61e6d69894799489df3fe3ee83b545..e94f97d3b940adf546a482695be82a7de075caaa 100644 (file)
@@ -72,12 +72,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip --version
-          pip install setuptools>=44 wheel>=0.34
+          pip install 'setuptools>=44' 'wheel>=0.34'
+          pip list
           pip wheel -w dist --no-use-pep517 -v --no-deps .
 
       - name: Install wheel
         # install the created wheel without using the pypi index
         run: |
+          pip install greenlet "importlib-metadata;python_version<'3.8'" &&
           pip install -f dist --no-index sqlalchemy
 
       - name: Check c extensions
@@ -196,7 +198,7 @@ jobs:
         # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux
         # the action uses the image for manylinux2010 but can generate also a manylinux1 wheel
         # change the tag of this image to change the image used
-        uses: RalfG/python-wheels-manylinux-build@v0.3-manylinux2010_x86_64
+        uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64
         # this action generates 3 wheels in dist/. linux, manylinux1 and manylinux2010
         with:
           # python-versions is the output of the previous step and is in the form <python tag>-<abi tag>. Eg cp37-cp37mu
@@ -214,9 +216,10 @@ jobs:
         # - check the c extension
         # - runs the tests
         run: |
-          pip install packaging>=20.4
+          pip install 'packaging>=20.4'
           if python .github/workflows/scripts/can_install.py "${{ matrix.python-version }}"
           then
+            pip install greenlet "importlib-metadata;python_version<'3.8'"
             pip install -f dist --no-index sqlalchemy
             python -c 'from sqlalchemy import cprocessors, cresultproxy, cutils'
             pip install pytest pytest-xdist ${{ matrix.extra-requires }}
@@ -281,3 +284,122 @@ jobs:
         run: |
           pip install -U twine
           twine upload --skip-existing dist/*manylinux*
+
+  make-wheel-linux-arm64:
+    name: ${{ matrix.python-version }}-arm64-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - "ubuntu-latest"
+        python-version:
+          # the versions are <python tag>-<abi tag> as specified in PEP 425.
+          - cp35-cp35m
+          - cp36-cp36m
+          - cp37-cp37m
+          - cp38-cp38
+
+      fail-fast: false
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Get python version
+        id: linux-py-version
+        env:
+          py_tag: ${{ matrix.python-version }}
+        # the command `echo "::set-output ...` is used to create an step output that can be used in following steps
+        # this is from https://github.community/t5/GitHub-Actions/Using-the-output-of-run-inside-of-if-condition/td-p/33920
+        run: |
+          version="${py_tag: 2:1}.${py_tag: 3:1}"
+          echo $version
+          echo "::set-output name=python-version::$version"
+
+      - name: Remove tag_build from setup.cfg
+        # sqlalchemy has `tag_build` set to `dev` in setup.cfg. We need to remove it before creating the weel
+        # otherwise it gets tagged with `dev0`
+        shell: pwsh
+        # This is equivalent to the sed commands:
+        # `sed -i '/tag_build=dev/d' setup.cfg`
+        # `sed -i '/tag_build = dev/d' setup.cfg`
+
+        # `-replace` uses a regexp match
+        # alternative form: `(get-content setup.cfg) | foreach-object{$_ -replace "tag_build.=.dev",""} | set-content setup.cfg`
+        run: |
+          (cat setup.cfg) | %{$_ -replace "tag_build.?=.?dev",""} | set-content setup.cfg
+
+      - name: Set up emulation
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Create wheel for manylinux2014
+        # this step uses the image provided by pypa here https://github.com/pypa/manylinux to generate the wheels on linux
+        # the action uses the image for manylinux2014 but can generate also a manylinux1 wheel
+        # change the tag of this image to change the image used
+        uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2014_aarch64
+        # this action generates 2 wheels in dist/. linux and manylinux2014
+        with:
+          # python-versions is the output of the previous step and is in the form <python tag>-<abi tag>. Eg cp37-cp37mu
+          python-versions: ${{ matrix.python-version }}
+          build-requirements: "setuptools>=44 wheel>=0.34"
+          # Create the wheel using --no-use-pep517 since locally we have pyproject
+          # This flag should be removed once sqlalchemy supports pep517
+          # `--no-deps` is used to only generate the wheel for the current library. Redundant in sqlalchemy since it has no dependencies
+          pip-wheel-args: "-w ./dist --no-use-pep517 -v --no-deps"
+
+      - name: Check created wheel
+        # check that the wheel is compatible with the current installation.
+        # - runs the tests
+        uses: docker://quay.io/pypa/manylinux2014_aarch64
+        with:
+          args: |
+            bash -c "
+            export PATH=/opt/python/${{ matrix.python-version }}/bin:$PATH &&
+            python --version &&
+            pip install greenlet \"importlib-metadata;python_version<'3.8'\" &&
+            pip install -f dist --no-index sqlalchemy &&
+            python -c 'from sqlalchemy import cprocessors, cresultproxy, cutils' &&
+            pip install pytest pytest-xdist ${{ matrix.extra-requires }} &&
+            pytest -n2 -q test -k 'not MockReconnectTest' --nomemory"
+
+      - name: Get wheel names
+        id: wheel-name
+        shell: bash
+        # the wheel creation step generates 2 wheels: linux and manylinux2014
+        # Pypi accepts only the manylinux versions
+        run: |
+          cd dist
+          echo ::set-output name=wheel2014::`ls *manylinux2014*`
+
+      - name: Upload wheel manylinux2014 to release
+        # upload the generated manylinux2014 wheel to the github release. Only a single file per step can be uploaded at the moment
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # this is a release to the event is called release: https://help.github.com/en/actions/reference/events-that-trigger-workflows#release-event-release
+          # the release event has the structure of this response https://developer.github.com/v3/repos/releases/#create-a-release
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: dist/${{ steps.wheel-name.outputs.wheel2014 }}
+          asset_name: ${{ steps.wheel-name.outputs.wheel2014 }}
+          asset_content_type: application/zip # application/octet-stream
+
+      - name: Set up Python for twine
+        # twine on py2 is very old and is no longer updated, so we change to python 3.8 before upload
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Publish wheel
+        # the action https://github.com/marketplace/actions/pypi-publish runs only on linux and we cannot specify
+        # additional options
+        # We upload manylinux2014 arm64 wheels. pip will download the appropriate one according to the system.
+        env:
+          TWINE_USERNAME: __token__
+          # replace TWINE_PASSWORD with token for real pypi
+          # TWINE_PASSWORD: ${{ secrets.test_pypi_token }}
+          TWINE_PASSWORD: ${{ secrets.pypi_token }}
+        run: |
+          pip install -U twine
+          twine upload --skip-existing dist/*manylinux*
index 53b4f5a0952cf5f8ba2192f9b8b308f32e9e1a33..52b238dabd7a848df788ba9770fe74cce5b6d14f 100644 (file)
@@ -14,7 +14,7 @@ env:
   TOX_WORKERS: -n2
 
 jobs:
-  run-test:
+  run-test-amd64:
     name: ${{ matrix.python-version }}-${{ matrix.build-type }}-${{ matrix.architecture }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,3 +51,34 @@ jobs:
 
       - name: Run tests
         run: tox -e github-${{ matrix.build-type }} -- -q --nomemory ${{ matrix.pytest-args }}
+
+  # Arm emulation is quite slow (~20min) so for now just run it when merging to master
+  # run-test-arm64:
+  #   name: ${{ matrix.python-version }}-${{ matrix.build-type }}-arm64-ubuntu-latest
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       python-version:
+  #         - "3.8"
+  #       build-type:
+  #         - "cext"
+  #         - "nocext"
+  #     fail-fast: true
+
+  #   steps:
+  #     - name: Checkout repo
+  #       uses: actions/checkout@v2
+
+  #     - name: Set up emulation
+  #       run: |
+  #         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+  #     - name: Run tests
+  #       uses: docker://arm64v8/python:3.8-buster
+  #       with:
+  #         args: |
+  #               bash -c "
+  #               python -m pip install --upgrade pip &&
+  #               pip install --upgrade tox setuptools &&
+  #               pip list &&
+  #               tox -e github-${{ matrix.build-type }} -- -q --nomemory ${{ matrix.pytest-args }}"
index 6a10b9626f189880cc8d665d74714924ef7605d9..89fc24b42a0b9367158dea0a1a72ab75621550ca 100644 (file)
@@ -87,3 +87,40 @@ jobs:
 
       - name: Run tests
         run: tox -e github-${{ matrix.build-type }} -- -q --nomemory ${{ matrix.pytest-args }}
+
+  run-test-arm64:
+    name: ${{ matrix.python-version }}-${{ matrix.build-type }}-arm64-ubuntu-latest
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - cp35-cp35m
+          - cp36-cp36m
+          - cp37-cp37m
+          - cp38-cp38
+        build-type:
+          - "cext"
+          - "nocext"
+
+      fail-fast: false
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Set up emulation
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Run tests
+        uses: docker://quay.io/pypa/manylinux2014_aarch64
+        with:
+          args: |
+            bash -c "
+            export PATH=/opt/python/${{ matrix.python-version }}/bin:$PATH &&
+            python --version &&
+            python -m pip install --upgrade pip &&
+            pip install --upgrade tox setuptools &&
+            pip list &&
+            tox -e github-${{ matrix.build-type }} -- -q --nomemory ${{ matrix.pytest-args }}
+            "