build: Revamp test/CI configuration

Reduce tox matrix to one env per python version, with two extra builds
for lint and docs. Delegate to tox from travis-ci.

Add 3.8 to testing. Simplify by dropping coverage reporting and
"no-deps" test runs.

diff --git a/.travis.yml b/.travis.yml
index 880a00c43e30f57547e86c66d4e2bfe6cf14e03c..374bd56425355048c9617de0fa4ca96c1bb17dea 100644 (file)
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,79 +17,31 @@ jobs:
     # "provisional feature" rules there are significant differences between patch
     # versions for asyncio and typing.
     - python: 3.5.2
+      env: TOX_ENV=py35-full
     - python: '3.5'
+      env: TOX_ENV=py35-full
     - python: '3.6'
+      env: TOX_ENV=py36-full
     - python: '3.7'
+      env: TOX_ENV=py37-full
+    - python: '3.8'
+      env: TOX_ENV=py38-full
     - python: pypy3.5-5.10.1
-      python: nightly
+      # Pypy is a lot slower due to jit warmup costs, so don't run the "full"
+      # test config there.
+      env: TOX_ENV=pypy3
+    - python: nightly
+      env: TOX_ENV=py3-full
+      # Docs and lint python versions must be synced with those in tox.ini
+    - python: '3.7'
+      env: TOX_ENV=docs
+    - python: '3.6'
+      env: TOX_ENV=lint
 
 install:
-    # On nightly, upgrade setuptools first to work around
-    # https://github.com/pypa/setuptools/issues/1257
-    - if [[ $TRAVIS_PYTHON_VERSION == 'nightly' ]]; then travis_retry pip install -U setuptools; fi
-    # TODO(bdarnell): pycares tests are currently disabled on travis due to ipv6 issues.
-    #- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycares; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycurl; fi
-    # Twisted is flaky on pypy (TODO: still? this note is quite old)
-    # It also sometimes has problems on nightly.
-    - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* && $TRAVIS_PYTHON_VERSION != 'nightly' ]]; then travis_retry pip install Twisted; fi
-    # Ideally we'd run the lint stuff on the latest Python
-    # version supported. But Python 3.7 requires slower-to-start VMs,
-    # so we run it on 3.6 to minimize total CI run time.
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then travis_retry pip install flake8 mypy==0.701 black==19.3b0; fi
-    # We run docs on py37 because we need some bug fixes for introspection of annotations.
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then travis_retry pip install -r docs/requirements.txt; fi
-    # On travis the extension should always be built
-    - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi
-    - travis_retry python setup.py install
-    - travis_retry pip install codecov virtualenv
-    # Create a separate no-dependencies virtualenv to make sure all imports
-    # of optional-dependencies are guarded.
-    - virtualenv /tmp/nodeps
-    - /tmp/nodeps/bin/python -VV
-    - /tmp/nodeps/bin/python setup.py install
+    - travis_retry pip install tox tox-venv
+    - python -VV
     - curl-config --version; pip freeze
 
 script:
-    # Run the tests once from the source directory to detect issues
-    # involving relative __file__ paths; see
-    # https://github.com/tornadoweb/tornado/issues/1780
-    - unset TORNADO_EXTENSION && python -m tornado.test
-    # For all other test variants, get out of the source directory before
-    # running tests to ensure that we get the installed speedups module
-    # instead of the source directory which doesn't have it.
-    - cd maint
-    # Copy the coveragerc down so coverage.py can find it.
-    - cp ../.coveragerc .
-    - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export TORNADO_EXTENSION=1; fi
-    - export TARGET="-m tornado.test.runtests"
-    # Travis workers are often overloaded and cause our tests to exceed
-    # the default timeout of 5s.
-    - export ASYNC_TEST_TIMEOUT=15
-    # We use "python -m coverage" instead of the "bin/coverage" script
-    # so we can pass additional arguments to python.
-    # coverage needs a function that was removed in python 3.6 so we can't
-    # run it with nightly cpython. Coverage is very slow on pypy.
-    - if [[ $TRAVIS_PYTHON_VERSION != nightly && $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then export RUN_COVERAGE=1; fi
-    - if [[ "$RUN_COVERAGE" == 1 ]]; then export TARGET="-m coverage run $TARGET"; fi
-    # See comments in tox.ini. Disabled on py35 because ResourceWarnings are too noisy there.
-    - if [[ $TRAVIS_PYTHON_VERSION != '3.5'* ]]; then export PYTHONWARNINGS=error,ignore:::site; fi
-    - python -bb $TARGET
-    - python -O $TARGET
-    - LANG=C python $TARGET
-    - LANG=en_US.utf-8 python $TARGET
-    #- if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then python $TARGET --resolver=tornado.platform.caresresolver.CaresResolver; fi
-    - if [[ $TRAVIS_PYTHON_VERSION != 'pypy3' ]]; then /tmp/nodeps/bin/python -m tornado.test.runtests; fi
-    # make coverage reports for Codecov to find
-    - if [[ "$RUN_COVERAGE" == 1 ]]; then coverage xml; fi
-    - export TORNADO_EXTENSION=0
-    - unset PYTHONWARNINGS
-    - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-out && sphinx-build -E -n -W -b html . sphinx-out); fi
-    - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then (cd ../docs && mkdir sphinx-doctest-out && sphinx-build -E -n -b doctest . sphinx-out); fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && flake8); fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && black --check --diff tornado demos); fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then (cd .. && mypy tornado); fi
-
-after_success:
-    # call codecov from project root
-    - if [[ "$RUN_COVERAGE" == 1 ]]; then cd ../ && codecov; fi
+    - "tox -e $TOX_ENV -- $TOX_ARGS"

diff --git a/tox.ini b/tox.ini
index 9a6f013a3855a5b8f6a79f33fed4ebaaf32688a5..c9e9460142ced47c64b172eb178da3863c57fbdc 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -11,108 +11,93 @@
 # 'port install curl +universal' to get both 32- and 64-bit versions of
 # libcurl.
 [tox]
+# When tox is run from a venv (instead of a virtualenv), it can get confused
+# unless tox-venv is also installed.
+requires = tox-venv
 envlist =
-        # Basic configurations: Run the tests in both minimal installations
-        # and with all optional dependencies.
-        # (pypy3 doesn't have any optional deps yet)
-        {py35,py36,py37,pypy3},
-        {py35,py36,py37}-full,
+        # Basic configurations: Run the tests for each python version.
+        py35-full,py36-full,py37-full,py38-full,pypy3-full
 
-        # Also run the tests with each possible replacement of a default
-        # component (only one 3.x version unless there are known differences).
+        # Build and test the docs with sphinx.
+        docs
 
-        # Alternate HTTP clients.
-        py3-curl,
+        # Run the linters.
+        lint
 
-        # Alternate Resolvers.
-        py3-full-caresresolver,
-
-        # Other configurations; see comments below.
-        py3-opt,
-        py3-{lang_c,lang_utf8},
-
-        # Ensure the sphinx build has no errors or warnings
-        py3-sphinx-docs,
-        # Run the doctests via sphinx (which covers things not run
-        # in the regular test suite and vice versa)
-        py3-sphinx-doctest,
-
-        py3-lint
-        py3-mypy
-
-# Tox doesn't like that we specify py3==py37
-ignore_basepython_conflict=True
+# Allow shell commands in tests
+whitelist_externals = /bin/sh
 
 [testenv]
-# Most of these are defaults, but if you specify any you can't fall back
-# defaults for the others.
 basepython =
+           py3: python3
            py35: python3.5
            py36: python3.6
            py37: python3.7
-           pypy: pypy
+           py38: python3.8
            pypy3: pypy3
-           py2: python2.7
-           py3: python3.7
+           # In theory, it doesn't matter which python version is used here.
+           # In practice, things like changes to the ast module can alter
+           # the outputs of the tools (especially where exactly the
+           # linter warning-supression comments go), so we specify a
+           # python version for these builds.
+           docs: python3.7
+           lint: python3.6
 
 deps =
-     # cpython-only deps: pycurl installs but curl_httpclient doesn't work;
-     # twisted mostly works but is a bit flaky under pypy.
-     {py35,py36,py37}-full: pycurl
-     py3: pycurl>=7.19.3.1
-     # twisted is cpython only.
-     {py35,py36,py37}-full: twisted
-     py3: twisted
-     {py3,py35,py36,py37}-full: pycares
-     sphinx: -r{toxinidir}/docs/requirements.txt
+     full: pycurl
+     full: twisted
+     full: pycares
+     docs: -r{toxinidir}/docs/requirements.txt
      lint: flake8
      lint: black==19.3b0
-     mypy: mypy==0.701
+     lint: mypy==0.701
 
 setenv =
-       # The extension is mandatory on cpython.
-       {py3,py35,py36,py37}: TORNADO_EXTENSION=1
-       # In python 3, opening files in text mode uses a
-       # system-dependent encoding by default.  Run the tests with "C"
-       # (ascii) and "utf-8" locales to ensure we don't have hidden
-       # dependencies on this setting.
-       lang_c: LANG=C
-       lang_utf8: LANG=en_US.utf-8
-       # tox's parser chokes if all the setenv entries are conditional.
-       DUMMY=dummy
-       {py3,py35,py36,py37}-no-ext: TORNADO_EXTENSION=0
+       # Treat the extension as mandatory in testing (but not on pypy)
+       {py3,py36,py37,py38}: TORNADO_EXTENSION=1
+       # CI workers are often overloaded and can cause our tests to exceed
+       # the default timeout of 5s.
+       ASYNC_TEST_TIMEOUT=15
+       # Treat warnings as errors by default. We have a whitelist of
+       # allowed warnings in runtests.py, but we want to be strict
+       # about any import-time warnings before that setup code is
+       # reached. Note that syntax warnings are only reported in
+       # -opt builds because regular builds reuse pycs created
+       # during sdist installation (and it doesn't seem to be
+       # possible to set environment variables during that phase of
+       # tox).
+       # ResourceWarnings are too noisy on py35 so don't enable 
+       # warnings-as-errors there.
+       {py3,py36,py37,py38,pypy3}: PYTHONWARNINGS=error:::tornado
+
 
 # All non-comment lines but the last must end in a backslash.
 # Tox filters line-by-line based on the environment name.
 commands =
-         python \
          # py3*: -b turns on an extra warning when calling
          # str(bytes), and -bb makes it an error.
-         -bb \
-         # Treat warnings as errors by default. We have a whitelist of
-         # allowed warnings in runtests.py, but we want to be strict
-         # about any import-time warnings before that setup code is
-         # reached. Note that syntax warnings are only reported in
-         # -opt builds because regular builds reuse pycs created
-         # during sdist installation (and it doesn't seem to be
-         # possible to set environment variables during that phase of
-         # tox).
-         "-Werror" \
-         # Virtualenv hits a deprecation warning very early which we must
-         # suppress here.
-         "-Wignore:::site" \
+         python -bb -m tornado.test {posargs:}
          # Python's optimized mode disables the assert statement, so
          # run the tests in this mode to ensure we haven't fallen into
          # the trap of relying on an assertion's side effects or using
          # them for things that should be runtime errors.
-         opt: -O \
-         -m tornado.test.runtests \
+         full: python -O -m tornado.test
+         # In python 3, opening files in text mode uses a
+         # system-dependent encoding by default.  Run the tests with "C"
+         # (ascii) and "utf-8" locales to ensure we don't have hidden
+         # dependencies on this setting.
+         full: sh -c 'LANG=C python -m tornado.test'
+         full: sh -c 'LANG=en_US.utf-8 python -m tornado.test'
          # Note that httpclient_test is always run with both client
          # implementations; this flag controls which client all the
          # other tests use.
-         curl: --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient \
-         caresresolver: --resolver=tornado.platform.caresresolver.CaresResolver \
-         {posargs:}
+         full: python -m tornado.test --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient
+         full: python -m tornado.test --resolver=tornado.platform.caresresolver.CaresResolver
+         # Run the tests once from the source directory to detect issues
+         # involving relative __file__ paths; see
+         # https://github.com/tornadoweb/tornado/issues/1780
+         full: sh -c '(cd {toxinidir} && unset TORNADO_EXTENSION && python -m tornado.test)'
+
 
 # python will import relative to the current working directory by default,
 # so cd into the tox working directory to avoid picking up the working
@@ -124,27 +109,21 @@ changedir = {toxworkdir}
 # Remove it (it's not a part of {opts}) to only install real releases.
 install_command = pip install {opts} {packages}
 
-[testenv:py3-sphinx-docs]
+[testenv:docs]
 changedir = docs
 # For some reason the extension fails to load in this configuration,
 # but it's not really needed for docs anyway.
 setenv = TORNADO_EXTENSION=0
 commands =
+    # Build the docs
     sphinx-build -q -E -n -W -b html . {envtmpdir}/html
+    # Run the doctests. No -W for doctests because that disallows tests
+    # with empty output.
+    sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest
 
-[testenv:py3-sphinx-doctest]
-changedir = docs
-setenv = TORNADO_EXTENSION=0
-# No -W for doctests because that disallows tests with empty output.
-commands =
-     sphinx-build -q -E -n -b doctest . {envtmpdir}/doctest
-
-[testenv:py3-lint]
+[testenv:lint]
 commands =
          flake8 {posargs:}
          black --check --diff {posargs:tornado demos}
-changedir = {toxinidir}
-
-[testenv:py3-mypy]
-commands = mypy {posargs:tornado}
+         mypy {posargs:tornado}
 changedir = {toxinidir}