From e145a3699214300af07e3a95e3b5c791bdce3416 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 3 Nov 2019 15:09:18 -0500 Subject: [PATCH] build: Revamp test/CI configuration Reduce tox matrix to one env per python version, with two extra builds for lint and docs. Delegate to tox from travis-ci. Add 3.8 to testing. Simplify by dropping coverage reporting and "no-deps" test runs. --- .travis.yml | 86 +++++++----------------------- tox.ini | 149 ++++++++++++++++++++++------------------------------ 2 files changed, 83 insertions(+), 152 deletions(-) diff --git a/.travis.yml b/.travis.yml index 880a00c43..374bd5642 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,79 +17,31 @@ jobs: # "provisional feature" rules there are significant differences between patch # versions for asyncio and typing. - python: 3.5.2 + env: TOX_ENV=py35-full - python: '3.5' + env: TOX_ENV=py35-full - python: '3.6' + env: TOX_ENV=py36-full - python: '3.7' + env: TOX_ENV=py37-full + - python: '3.8' + env: TOX_ENV=py38-full - python: pypy3.5-5.10.1 - python: nightly + # Pypy is a lot slower due to jit warmup costs, so don't run the "full" + # test config there. + env: TOX_ENV=pypy3 + - python: nightly + env: TOX_ENV=py3-full + # Docs and lint python versions must be synced with those in tox.ini + - python: '3.7' + env: TOX_ENV=docs + - python: '3.6' + env: TOX_ENV=lint install: - # On nightly, upgrade setuptools first to work around - # https://github.com/pypa/setuptools/issues/1257 - - if [[ $TRAVIS_PYTHON_VERSION == 'nightly' ]]; then travis_retry pip install -U setuptools; fi - # TODO(bdarnell): pycares tests are currently disabled on travis due to ipv6 issues. - #- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycares; fi - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycurl; fi - # Twisted is flaky on pypy (TODO: still? this note is quite old) - # It also sometimes has problems on nightly. - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* && $TRAVIS_PYTHON_VERSION != 'nightly' ]]; then travis_retry pip install Twisted; fi - # Ideally we'd run the lint stuff on the latest Python - # version supported. But Python 3.7 requires slower-to-start VMs, - # so we run it on 3.6 to minimize total CI run time. - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then travis_retry pip install flake8 mypy==0.701 black==19.3b0; fi - # We run docs on py37 because we need some bug fixes for introspection of annotations. - - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then travis_retry pip install -r docs/requirements.txt; fi - # On travis the extension should always be built - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi - - travis_retry python setup.py install - - travis_retry pip install codecov virtualenv - # Create a separate no-dependencies virtualenv to make sure all imports - # of optional-dependencies are guarded. - - virtualenv /tmp/nodeps - - /tmp/nodeps/bin/python -VV - - /tmp/nodeps/bin/python setup.py install + - travis_retry pip install tox tox-venv + - python -VV - curl-config --version; pip freeze script: - # Run the tests once from the source directory to detect issues - # involving relative __file__ paths; see - # https://github.com/tornadoweb/tornado/issues/1780 - - unset TORNADO_EXTENSION && python -m tornado.test - # For all other test variants, get out of the source directory before - # running tests to ensure that we get the installed speedups module - # instead of the source directory which doesn't have it. - - cd maint - # Copy the coveragerc down so coverage.py can find it. - - cp ../.coveragerc . - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi - - export TARGET="-m tornado.test.runtests" - # Travis workers are often overloaded and cause our tests to exceed - # the default timeout of 5s. - - export ASYNC_TEST_TIMEOUT=15 - # We use "python -m coverage" instead of the "bin/coverage" script - # so we can pass additional arguments to python. - # coverage needs a function that was removed in python 3.6 so we can't - # run it with nightly cpython. Coverage is very slow on pypy. - - if [[ $TRAVIS_PYTHON_VERSION != nightly && $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export RUN_COVERAGE=1; fi - - if [[ "$RUN_COVERAGE" == 1 ]]; then export TARGET="-m coverage run $TARGET"; fi - # See comments in tox.ini. Disabled on py35 because ResourceWarnings are too noisy there. - - if [[ $TRAVIS_PYTHON_VERSION != '3.5'* ]]; then export PYTHONWARNINGS=error,ignore:::site; fi - - python -bb $TARGET - - python -O $TARGET - - LANG=C python $TARGET - - LANG=en_US.utf-8 python $TARGET - #- if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then python $TARGET --resolver=tornado.platform.caresresolver.CaresResolver; fi - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy3' ]]; then /tmp/nodeps/bin/python -m tornado.test.runtests; fi - # make coverage reports for Codecov to find - - if [[ "$RUN_COVERAGE" == 1 ]]; then coverage xml; fi - - export TORNADO_EXTENSION=0 - - unset PYTHONWARNINGS - - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-out && sphinx-build -E -n -W -b html . sphinx-out); fi - - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-doctest-out && sphinx-build -E -n -b doctest . sphinx-out); fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && flake8); fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && black --check --diff tornado demos); fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && mypy tornado); fi - -after_success: - # call codecov from project root - - if [[ "$RUN_COVERAGE" == 1 ]]; then cd ../ && codecov; fi + - "tox -e $TOX_ENV -- $TOX_ARGS" diff --git a/tox.ini b/tox.ini index 9a6f013a3..c9e946014 100644 --- a/tox.ini +++ b/tox.ini @@ -11,108 +11,93 @@ # 'port install curl +universal' to get both 32- and 64-bit versions of # libcurl. [tox] +# When tox is run from a venv (instead of a virtualenv), it can get confused +# unless tox-venv is also installed. +requires = tox-venv envlist = - # Basic configurations: Run the tests in both minimal installations - # and with all optional dependencies. - # (pypy3 doesn't have any optional deps yet) - {py35,py36,py37,pypy3}, - {py35,py36,py37}-full, + # Basic configurations: Run the tests for each python version. + py35-full,py36-full,py37-full,py38-full,pypy3-full - # Also run the tests with each possible replacement of a default - # component (only one 3.x version unless there are known differences). + # Build and test the docs with sphinx. + docs - # Alternate HTTP clients. - py3-curl, + # Run the linters. + lint - # Alternate Resolvers. - py3-full-caresresolver, - - # Other configurations; see comments below. - py3-opt, - py3-{lang_c,lang_utf8}, - - # Ensure the sphinx build has no errors or warnings - py3-sphinx-docs, - # Run the doctests via sphinx (which covers things not run - # in the regular test suite and vice versa) - py3-sphinx-doctest, - - py3-lint - py3-mypy - -# Tox doesn't like that we specify py3==py37 -ignore_basepython_conflict=True +# Allow shell commands in tests +whitelist_externals = /bin/sh [testenv] -# Most of these are defaults, but if you specify any you can't fall back -# defaults for the others. basepython = + py3: python3 py35: python3.5 py36: python3.6 py37: python3.7 - pypy: pypy + py38: python3.8 pypy3: pypy3 - py2: python2.7 - py3: python3.7 + # In theory, it doesn't matter which python version is used here. + # In practice, things like changes to the ast module can alter + # the outputs of the tools (especially where exactly the + # linter warning-supression comments go), so we specify a + # python version for these builds. + docs: python3.7 + lint: python3.6 deps = - # cpython-only deps: pycurl installs but curl_httpclient doesn't work; - # twisted mostly works but is a bit flaky under pypy. - {py35,py36,py37}-full: pycurl - py3: pycurl>=7.19.3.1 - # twisted is cpython only. - {py35,py36,py37}-full: twisted - py3: twisted - {py3,py35,py36,py37}-full: pycares - sphinx: -r{toxinidir}/docs/requirements.txt + full: pycurl + full: twisted + full: pycares + docs: -r{toxinidir}/docs/requirements.txt lint: flake8 lint: black==19.3b0 - mypy: mypy==0.701 + lint: mypy==0.701 setenv = - # The extension is mandatory on cpython. - {py3,py35,py36,py37}: TORNADO_EXTENSION=1 - # In python 3, opening files in text mode uses a - # system-dependent encoding by default. Run the tests with "C" - # (ascii) and "utf-8" locales to ensure we don't have hidden - # dependencies on this setting. - lang_c: LANG=C - lang_utf8: LANG=en_US.utf-8 - # tox's parser chokes if all the setenv entries are conditional. - DUMMY=dummy - {py3,py35,py36,py37}-no-ext: TORNADO_EXTENSION=0 + # Treat the extension as mandatory in testing (but not on pypy) + {py3,py36,py37,py38}: TORNADO_EXTENSION=1 + # CI workers are often overloaded and can cause our tests to exceed + # the default timeout of 5s. + ASYNC_TEST_TIMEOUT=15 + # Treat warnings as errors by default. We have a whitelist of + # allowed warnings in runtests.py, but we want to be strict + # about any import-time warnings before that setup code is + # reached. Note that syntax warnings are only reported in + # -opt builds because regular builds reuse pycs created + # during sdist installation (and it doesn't seem to be + # possible to set environment variables during that phase of + # tox). + # ResourceWarnings are too noisy on py35 so don't enable + # warnings-as-errors there. + {py3,py36,py37,py38,pypy3}: PYTHONWARNINGS=error:::tornado + # All non-comment lines but the last must end in a backslash. # Tox filters line-by-line based on the environment name. commands = - python \ # py3*: -b turns on an extra warning when calling # str(bytes), and -bb makes it an error. - -bb \ - # Treat warnings as errors by default. We have a whitelist of - # allowed warnings in runtests.py, but we want to be strict - # about any import-time warnings before that setup code is - # reached. Note that syntax warnings are only reported in - # -opt builds because regular builds reuse pycs created - # during sdist installation (and it doesn't seem to be - # possible to set environment variables during that phase of - # tox). - "-Werror" \ - # Virtualenv hits a deprecation warning very early which we must - # suppress here. - "-Wignore:::site" \ + python -bb -m tornado.test {posargs:} # Python's optimized mode disables the assert statement, so # run the tests in this mode to ensure we haven't fallen into # the trap of relying on an assertion's side effects or using # them for things that should be runtime errors. - opt: -O \ - -m tornado.test.runtests \ + full: python -O -m tornado.test + # In python 3, opening files in text mode uses a + # system-dependent encoding by default. Run the tests with "C" + # (ascii) and "utf-8" locales to ensure we don't have hidden + # dependencies on this setting. + full: sh -c 'LANG=C python -m tornado.test' + full: sh -c 'LANG=en_US.utf-8 python -m tornado.test' # Note that httpclient_test is always run with both client # implementations; this flag controls which client all the # other tests use. - curl: --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient \ - caresresolver: --resolver=tornado.platform.caresresolver.CaresResolver \ - {posargs:} + full: python -m tornado.test --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient + full: python -m tornado.test --resolver=tornado.platform.caresresolver.CaresResolver + # Run the tests once from the source directory to detect issues + # involving relative __file__ paths; see + # https://github.com/tornadoweb/tornado/issues/1780 + full: sh -c '(cd {toxinidir} && unset TORNADO_EXTENSION && python -m tornado.test)' + # python will import relative to the current working directory by default, # so cd into the tox working directory to avoid picking up the working @@ -124,27 +109,21 @@ changedir = {toxworkdir} # Remove it (it's not a part of {opts}) to only install real releases. install_command = pip install {opts} {packages} -[testenv:py3-sphinx-docs] +[testenv:docs] changedir = docs # For some reason the extension fails to load in this configuration, # but it's not really needed for docs anyway. setenv = TORNADO_EXTENSION=0 commands = + # Build the docs sphinx-build -q -E -n -W -b html . {envtmpdir}/html + # Run the doctests. No -W for doctests because that disallows tests + # with empty output. + sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest -[testenv:py3-sphinx-doctest] -changedir = docs -setenv = TORNADO_EXTENSION=0 -# No -W for doctests because that disallows tests with empty output. -commands = - sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest - -[testenv:py3-lint] +[testenv:lint] commands = flake8 {posargs:} black --check --diff {posargs:tornado demos} -changedir = {toxinidir} - -[testenv:py3-mypy] -commands = mypy {posargs:tornado} + mypy {posargs:tornado} changedir = {toxinidir} -- 2.47.2