From: Mike Bayer Date: Tue, 25 Jan 2022 05:45:30 +0000 (-0500) Subject: replace test tags with pytest.mark X-Git-Tag: rel_2_0_0b1~512^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ca48f461b2dcac2970829e4e021316654c308d90;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git replace test tags with pytest.mark replaced the __tags__ class attribute and the --exclude-tags / --include-tags test runner options with regular pytest.mark names so that we can take advantage of mark expressions. options --nomemory, --notimingintensive, --backend-only, --exclude-tags, --include-tags remain as legacy but make use of pytest mark for implemementation. Added a "mypy" mark for the section of tests that are doing mypy integration tests. The __backend__ and __sparse_backend__ class attributes also use pytest marks for their implementation, which also allows the marks "backend" and "sparse_backend" to be used explicitly. Also removed the no longer used "--cdecimal" option as this was python 2 specific. in theory, the usage of pytest marks could expand such that the whole exclusions system would be based on it, but this does not seem to have any advantage at the moment. Change-Id: Ideeb57d9d49f0efc7fc0b6b923b31207ab783025 --- diff --git a/lib/sqlalchemy/testing/__init__.py b/lib/sqlalchemy/testing/__init__.py index fd6ddf5931..4253aa61bc 100644 --- a/lib/sqlalchemy/testing/__init__.py +++ b/lib/sqlalchemy/testing/__init__.py @@ -42,6 +42,7 @@ from .assertions import not_in from .assertions import not_in_ from .assertions import startswith_ from .assertions import uses_deprecated +from .config import add_to_marker from .config import async_test from .config import combinations from .config import combinations_list diff --git a/lib/sqlalchemy/testing/config.py b/lib/sqlalchemy/testing/config.py index f326c124d4..268a564215 100644 --- a/lib/sqlalchemy/testing/config.py +++ b/lib/sqlalchemy/testing/config.py @@ -106,6 +106,14 @@ def mark_base_test_class(): return _fixture_functions.mark_base_test_class() +class _AddToMarker: + def __getattr__(self, attr): + return getattr(_fixture_functions.add_to_marker, attr) + + +add_to_marker = _AddToMarker() + + class Config: def __init__(self, db, db_opts, options, file_config): self._set_name(db) diff --git a/lib/sqlalchemy/testing/exclusions.py b/lib/sqlalchemy/testing/exclusions.py index b92d6859fd..b51f6e57c5 100644 --- a/lib/sqlalchemy/testing/exclusions.py +++ b/lib/sqlalchemy/testing/exclusions.py @@ -35,7 +35,6 @@ class compound: def __init__(self): self.fails = set() self.skips = set() - self.tags = set() def __add__(self, other): return self.add(other) @@ -44,25 +43,22 @@ class compound: rule = compound() rule.skips.update(self.skips) rule.skips.update(self.fails) - rule.tags.update(self.tags) return rule def add(self, *others): copy = compound() copy.fails.update(self.fails) copy.skips.update(self.skips) - copy.tags.update(self.tags) + for other in others: copy.fails.update(other.fails) copy.skips.update(other.skips) - copy.tags.update(other.tags) return copy def not_(self): copy = compound() copy.fails.update(NotPredicate(fail) for fail in self.fails) copy.skips.update(NotPredicate(skip) for skip in self.skips) - copy.tags.update(self.tags) return copy @property @@ -83,16 +79,9 @@ class compound: if predicate(config) ] - def include_test(self, include_tags, exclude_tags): - return bool( - not self.tags.intersection(exclude_tags) - and (not include_tags or self.tags.intersection(include_tags)) - ) - def _extend(self, other): self.skips.update(other.skips) self.fails.update(other.fails) - self.tags.update(other.tags) def __call__(self, fn): if hasattr(fn, "_sa_exclusion_extend"): @@ -166,16 +155,6 @@ class compound: ) -def requires_tag(tagname): - return tags([tagname]) - - -def tags(tagnames): - comp = compound() - comp.tags.update(tagnames) - return comp - - def only_if(predicate, reason=None): predicate = _as_predicate(predicate) return skip_if(NotPredicate(predicate), reason) diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 5a4bfe3a64..0b4451b3c8 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -106,21 +106,28 @@ def setup_options(make_option): ) make_option( "--backend-only", - action="store_true", - dest="backend_only", - help="Run only tests marked with __backend__ or __sparse_backend__", + action="callback", + zeroarg_callback=_set_tag_include("backend"), + help=( + "Run only tests marked with __backend__ or __sparse_backend__; " + "this is now equivalent to the pytest -m backend mark expression" + ), ) make_option( "--nomemory", - action="store_true", - dest="nomemory", - help="Don't run memory profiling tests", + action="callback", + zeroarg_callback=_set_tag_exclude("memory_intensive"), + help="Don't run memory profiling tests; " + "this is now equivalent to the pytest -m 'not memory_intensive' " + "mark expression", ) make_option( "--notimingintensive", - action="store_true", - dest="notimingintensive", - help="Don't run timing intensive tests", + action="callback", + zeroarg_callback=_set_tag_exclude("timing_intensive"), + help="Don't run timing intensive tests; " + "this is now equivalent to the pytest -m 'not timing_intensive' " + "mark expression", ) make_option( "--profile-sort", @@ -170,27 +177,21 @@ def setup_options(make_option): callback=_requirements_opt, help="requirements class for testing, overrides setup.cfg", ) - make_option( - "--with-cdecimal", - action="store_true", - dest="cdecimal", - default=False, - help="Monkeypatch the cdecimal library into Python 'decimal' " - "for all tests", - ) make_option( "--include-tag", action="callback", callback=_include_tag, type=str, - help="Include tests with tag ", + help="Include tests with tag ; " + "legacy, use pytest -m 'tag' instead", ) make_option( "--exclude-tag", action="callback", callback=_exclude_tag, type=str, - help="Exclude tests with tag ", + help="Exclude tests with tag ; " + "legacy, use pytest -m 'not tag' instead", ) make_option( "--write-profiles", @@ -240,15 +241,9 @@ def memoize_important_follower_config(dict_): This invokes in the parent process after normal config is set up. - This is necessary as pytest seems to not be using forking, so we - start with nothing in memory, *but* it isn't running our argparse - callables, so we have to just copy all of that over. + Hook is currently not used. """ - dict_["memoized_config"] = { - "include_tags": include_tags, - "exclude_tags": exclude_tags, - } def restore_important_follower_config(dict_): @@ -256,10 +251,9 @@ def restore_important_follower_config(dict_): This invokes in the follower process. + Hook is currently not used. + """ - global include_tags, exclude_tags - include_tags.update(dict_["memoized_config"]["include_tags"]) - exclude_tags.update(dict_["memoized_config"]["exclude_tags"]) def read_config(): @@ -322,6 +316,20 @@ def _requirements_opt(opt_str, value, parser): _setup_requirements(value) +def _set_tag_include(tag): + def _do_include_tag(opt_str, value, parser): + _include_tag(opt_str, tag, parser) + + return _do_include_tag + + +def _set_tag_exclude(tag): + def _do_exclude_tag(opt_str, value, parser): + _exclude_tag(opt_str, tag, parser) + + return _do_exclude_tag + + def _exclude_tag(opt_str, value, parser): exclude_tags.add(value.replace("-", "_")) @@ -350,26 +358,6 @@ def _setup_options(opt, file_config): options = opt -@pre -def _set_nomemory(opt, file_config): - if opt.nomemory: - exclude_tags.add("memory_intensive") - - -@pre -def _set_notimingintensive(opt, file_config): - if opt.notimingintensive: - exclude_tags.add("timing_intensive") - - -@pre -def _monkeypatch_cdecimal(options, file_config): - if options.cdecimal: - import cdecimal - - sys.modules["decimal"] = cdecimal - - @post def __ensure_cext(opt, file_config): if os.environ.get("REQUIRE_SQLALCHEMY_CEXT", "0") == "1": @@ -515,13 +503,6 @@ def want_class(name, cls): return False elif name.startswith("_"): return False - elif ( - config.options.backend_only - and not getattr(cls, "__backend__", False) - and not getattr(cls, "__sparse_backend__", False) - and not getattr(cls, "__only_on__", False) - ): - return False else: return True @@ -531,33 +512,13 @@ def want_method(cls, fn): return False elif fn.__module__ is None: return False - elif include_tags: - return ( - hasattr(cls, "__tags__") - and exclusions.tags(cls.__tags__).include_test( - include_tags, exclude_tags - ) - ) or ( - hasattr(fn, "_sa_exclusion_extend") - and fn._sa_exclusion_extend.include_test( - include_tags, exclude_tags - ) - ) - elif exclude_tags and hasattr(cls, "__tags__"): - return exclusions.tags(cls.__tags__).include_test( - include_tags, exclude_tags - ) - elif exclude_tags and hasattr(fn, "_sa_exclusion_extend"): - return fn._sa_exclusion_extend.include_test(include_tags, exclude_tags) else: return True -def generate_sub_tests(cls, module): - if getattr(cls, "__backend__", False) or getattr( - cls, "__sparse_backend__", False - ): - sparse = getattr(cls, "__sparse_backend__", False) +def generate_sub_tests(cls, module, markers): + if "backend" in markers or "sparse_backend" in markers: + sparse = "sparse_backend" in markers for cfg in _possible_configs_for_cls(cls, sparse=sparse): orig_name = cls.__name__ @@ -780,6 +741,10 @@ class FixtureFunctions(abc.ABC): def mark_base_test_class(self): raise NotImplementedError() + @abc.abstractproperty + def add_to_marker(self): + raise NotImplementedError() + _fixture_fn_class = None diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index 2ae6730bbe..363a73eccf 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -69,6 +69,17 @@ def pytest_addoption(parser): def pytest_configure(config): + if plugin_base.exclude_tags or plugin_base.include_tags: + if config.option.markexpr: + raise ValueError( + "Can't combine explicit pytest marks with legacy options " + "such as --backend-only, --exclude-tags, etc. " + ) + config.option.markexpr = " and ".join( + list(plugin_base.include_tags) + + [f"not {tag}" for tag in plugin_base.exclude_tags] + ) + if config.pluginmanager.hasplugin("xdist"): config.pluginmanager.register(XDistHooks()) @@ -206,18 +217,43 @@ def pytest_collection_modifyitems(session, config, items): def setup_test_classes(): for test_class in test_classes: + + # transfer legacy __backend__ and __sparse_backend__ symbols + # to be markers + add_markers = set() + if getattr(test_class.cls, "__backend__", False) or getattr( + test_class.cls, "__only_on__", False + ): + add_markers = {"backend"} + elif getattr(test_class.cls, "__sparse_backend__", False): + add_markers = {"sparse_backend"} + else: + add_markers = frozenset() + + existing_markers = { + mark.name for mark in test_class.iter_markers() + } + add_markers = add_markers - existing_markers + all_markers = existing_markers.union(add_markers) + + for marker in add_markers: + test_class.add_marker(marker) + for sub_cls in plugin_base.generate_sub_tests( - test_class.cls, test_class.module + test_class.cls, test_class.module, all_markers ): if sub_cls is not test_class.cls: per_cls_dict = rebuilt_items[test_class.cls] module = test_class.getparent(pytest.Module) - for fn in collect( - pytest.Class.from_parent( - name=sub_cls.__name__, parent=module - ) - ): + + new_cls = pytest.Class.from_parent( + name=sub_cls.__name__, parent=module + ) + for marker in add_markers: + new_cls.add_marker(marker) + + for fn in collect(new_cls): per_cls_dict[fn.name].append(fn) # class requirements will sometimes need to access the DB to check @@ -573,6 +609,10 @@ class PytestFixtureFunctions(plugin_base.FixtureFunctions): def skip_test_exception(self, *arg, **kw): return pytest.skip.Exception(*arg, **kw) + @property + def add_to_marker(self): + return pytest.mark + def mark_base_test_class(self): return pytest.mark.usefixtures( "setup_class_methods", "setup_test_methods" diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index 6af2687a9b..410ab26edc 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -17,6 +17,7 @@ to provide specific inclusion/exclusions. import platform +from . import config from . import exclusions from . import only_on from .. import create_engine @@ -1295,11 +1296,11 @@ class SuiteRequirements(Requirements): @property def timing_intensive(self): - return exclusions.requires_tag("timing_intensive") + return config.add_to_marker.timing_intensive @property def memory_intensive(self): - return exclusions.requires_tag("memory_intensive") + return config.add_to_marker.memory_intensive @property def threading_with_mock(self): diff --git a/pyproject.toml b/pyproject.toml index 036892d45b..042bab6bff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ target-version = ['py37'] [tool.pytest.ini_options] -addopts = "--tb native -v -r sfxX --maxfail=250 -p warnings -p logging" +addopts = "--tb native -v -r sfxX --maxfail=250 -p warnings -p logging --strict-markers" python_files = "test/*test_*.py" minversion = "6.2" filterwarnings = [ @@ -23,7 +23,13 @@ filterwarnings = [ "error::DeprecationWarning:test", "error::DeprecationWarning:sqlalchemy" ] - +markers = [ + "memory_intensive: memory / CPU intensive suite tests", + "mypy: mypy integration / plugin tests", + "timing_intensive: time-oriented tests that are sensitive to race conditions", + "backend: tests that should run on all backends; typically dialect-sensitive", + "sparse_backend: tests that should run on multiple backends, not necessarily all", +] [tool.pyright] include = [ diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py index 2b806baf7a..2fc61706cf 100644 --- a/test/aaa_profiling/test_memusage.py +++ b/test/aaa_profiling/test_memusage.py @@ -252,8 +252,8 @@ class EnsureZeroed(fixtures.ORMTest): ) +@testing.add_to_marker.memory_intensive class MemUsageTest(EnsureZeroed): - __tags__ = ("memory_intensive",) __requires__ = ("cpython", "no_windows") def test_type_compile(self): @@ -347,9 +347,8 @@ class MemUsageTest(EnsureZeroed): go() +@testing.add_to_marker.memory_intensive class MemUsageWBackendTest(fixtures.MappedTest, EnsureZeroed): - - __tags__ = ("memory_intensive",) __requires__ = "cpython", "memory_process_intensive", "no_asyncio" __sparse_backend__ = True @@ -1150,8 +1149,8 @@ class MemUsageWBackendTest(fixtures.MappedTest, EnsureZeroed): metadata.drop_all(self.engine) +@testing.add_to_marker.memory_intensive class CycleTest(_fixtures.FixtureTest): - __tags__ = ("memory_intensive",) __requires__ = ("cpython", "no_windows") run_setup_mappers = "once" diff --git a/test/ext/mypy/test_mypy_plugin_py3k.py b/test/ext/mypy/test_mypy_plugin_py3k.py index 681c9d57ba..cc8d8955f6 100644 --- a/test/ext/mypy/test_mypy_plugin_py3k.py +++ b/test/ext/mypy/test_mypy_plugin_py3k.py @@ -10,6 +10,7 @@ from sqlalchemy.testing import eq_ from sqlalchemy.testing import fixtures +@testing.add_to_marker.mypy class MypyPluginTest(fixtures.TestBase): __requires__ = ("sqlalchemy2_stubs",) diff --git a/tox.ini b/tox.ini index 2100aa507e..3e0c3496f1 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,8 @@ allowlist_externals=sh setenv= PYTHONPATH= PYTHONNOUSERSITE=1 - MEMUSAGE=--nomemory + PYTEST_EXCLUDES=-m "not memory_intensive and not mypy" + BASECOMMAND=python -m pytest --rootdir {toxinidir} --log-info=sqlalchemy.testing WORKERS={env:TOX_WORKERS:-n4 --max-worker-restart=5} @@ -85,8 +86,8 @@ setenv= nocext: DISABLE_SQLALCHEMY_CEXT=1 cext: REQUIRE_SQLALCHEMY_CEXT=1 cov: COVERAGE={[testenv]cov_args} - backendonly: BACKENDONLY=--backend-only - memusage: MEMUSAGE='-k test_memusage' + backendonly: PYTEST_EXCLUDES="-m backend" + memusage: PYTEST_EXCLUDES="-m memory_intensive" oracle: WORKERS={env:TOX_WORKERS:-n2 --max-worker-restart=5} oracle: ORACLE={env:TOX_ORACLE:--db oracle} @@ -111,7 +112,6 @@ setenv= mssql: MSSQL={env:TOX_MSSQL:--db mssql} oracle,mssql,sqlite_file: IDENTS=--write-idents db_idents.txt - oracle,mssql,sqlite_file: MEMUSAGE=--nomemory # tox as of 2.0 blocks all environment variables from the # outside, unless they are here (or in TOX_TESTENV_PASSENV, @@ -124,7 +124,7 @@ commands= # that flag for coverage mode. nocext: sh -c "rm -f lib/sqlalchemy/*.so" - {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:EXTRA_SQLITE_DRIVERS:} {env:POSTGRESQL:} {env:EXTRA_PG_DRIVERS:} {env:MYSQL:} {env:EXTRA_MYSQL_DRIVERS:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:MEMUSAGE:} {env:COVERAGE:} {posargs} + {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:EXTRA_SQLITE_DRIVERS:} {env:POSTGRESQL:} {env:EXTRA_PG_DRIVERS:} {env:MYSQL:} {env:EXTRA_MYSQL_DRIVERS:} {env:ORACLE:} {env:MSSQL:} {env:IDENTS:} {env:PYTEST_EXCLUDES:} {env:COVERAGE:} {posargs} oracle,mssql,sqlite_file: python reap_dbs.py db_idents.txt @@ -148,7 +148,7 @@ deps= patch==1.* git+https://github.com/sqlalchemy/sqlalchemy2-stubs commands = - pytest test/ext/mypy/test_mypy_plugin_py3k.py {posargs} + pytest -m mypy {posargs} # thanks to https://julien.danjou.info/the-best-flake8-extensions/ [testenv:pep8] @@ -174,7 +174,7 @@ commands = deps = {[testenv]deps} .[aiosqlite] commands= - python -m pytest {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:MEMUSAGE:} {env:COVERAGE:} {posargs} + python -m pytest {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:IDENTS:} {env:PYTEST_EXCLUDES:} {env:COVERAGE:} {posargs} oracle,mssql,sqlite_file: python reap_dbs.py db_idents.txt # command run in the github action when cext are not active. @@ -182,5 +182,5 @@ commands= deps = {[testenv]deps} .[aiosqlite] commands= - python -m pytest {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:MEMUSAGE:} {env:COVERAGE:} {posargs} + python -m pytest {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:IDENTS:} {env:PYTEST_EXCLUDES:} {env:COVERAGE:} {posargs} oracle,mssql,sqlite_file: python reap_dbs.py db_idents.txt