From: Mike Bayer Date: Tue, 22 Aug 2017 02:26:53 +0000 (-0400) Subject: - backport all pytest-xdist / multi backend changes X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dcb352589c046293325c786a5967892ac55496b6;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - backport all pytest-xdist / multi backend changes from 1.2 master back to 1.0 for CI Change-Id: I31719ccececd474e81e765dc3a8896f89f5a84eb --- diff --git a/lib/sqlalchemy/testing/config.py b/lib/sqlalchemy/testing/config.py index 592f333c24..73d8a7de88 100644 --- a/lib/sqlalchemy/testing/config.py +++ b/lib/sqlalchemy/testing/config.py @@ -15,11 +15,16 @@ file_config = None test_schema = None test_schema_2 = None _current = None -_skip_test_exception = None + +try: + from unittest import SkipTest as _skip_test_exception +except ImportError: + _skip_test_exception = None class Config(object): def __init__(self, db, db_opts, options, file_config): + self._set_name(db) self.db = db self.db_opts = db_opts self.options = options @@ -28,7 +33,14 @@ class Config(object): self.test_schema_2 = "test_schema_2" _stack = collections.deque() - _configs = {} + _configs = set() + + def _set_name(self, db): + if db.dialect.server_version_info: + svi = ".".join(str(tok) for tok in db.dialect.server_version_info) + self.name = "%s+%s_[%s]" % (db.name, db.driver, svi) + else: + self.name = "%s+%s" % (db.name, db.driver) @classmethod def register(cls, db, db_opts, options, file_config): @@ -38,10 +50,7 @@ class Config(object): gets set as the "_current". """ cfg = Config(db, db_opts, options, file_config) - - cls._configs[cfg.db.name] = cfg - cls._configs[(cfg.db.name, cfg.db.dialect)] = cfg - cls._configs[cfg.db] = cfg + cls._configs.add(cfg) return cfg @classmethod @@ -76,8 +85,7 @@ class Config(object): @classmethod def all_configs(cls): - for cfg in set(cls._configs.values()): - yield cfg + return cls._configs @classmethod def all_dbs(cls): @@ -90,3 +98,4 @@ class Config(object): def skip_test(msg): raise _skip_test_exception(msg) + diff --git a/lib/sqlalchemy/testing/distutils_run.py b/lib/sqlalchemy/testing/distutils_run.py deleted file mode 100644 index 38de8872cb..0000000000 --- a/lib/sqlalchemy/testing/distutils_run.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Quick and easy way to get setup.py test to run py.test without any -custom setuptools/distutils code. - -""" -import unittest -import pytest - - -class TestSuite(unittest.TestCase): - def test_sqlalchemy(self): - pytest.main(["-n", "4", "-q"]) diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 70d8ad5f77..ca8abaa8e3 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -63,6 +63,8 @@ def setup_options(make_option): help="Drop all tables in the target database first") make_option("--backend-only", action="store_true", dest="backend_only", help="Run only tests marked with __backend__") + make_option("--nomemory", action="store_true", dest="nomemory", + help="Don't run memory profiling tests") make_option("--low-connections", action="store_true", dest="low_connections", help="Use a low number of distinct connections - " @@ -177,6 +179,7 @@ def post_begin(): warnings.setup_filters() + def _log(opt_str, value, parser): global logging if not logging: @@ -227,6 +230,12 @@ def _setup_options(opt, file_config): options = opt +@pre +def _set_nomemory(opt, file_config): + if opt.nomemory: + exclude_tags.add("memory_intensive") + + @pre def _monkeypatch_cdecimal(options, file_config): if options.cdecimal: @@ -266,7 +275,13 @@ def _engine_uri(options, file_config): if not db_urls: db_urls.append(file_config.get('db', 'default')) + config._current = None for db_url in db_urls: + + if options.write_idents and provision.FOLLOWER_IDENT: # != 'master': + with open(options.write_idents, "a") as file_: + file_.write(provision.FOLLOWER_IDENT + " " + db_url + "\n") + cfg = provision.setup_config( db_url, options, file_config, provision.FOLLOWER_IDENT) @@ -407,12 +422,21 @@ def want_method(cls, fn): def generate_sub_tests(cls, module): if getattr(cls, '__backend__', False): for cfg in _possible_configs_for_cls(cls): - name = "%s_%s_%s" % (cls.__name__, cfg.db.name, cfg.db.driver) + orig_name = cls.__name__ + + # we can have special chars in these names except for the + # pytest junit plugin, which is tripped up by the brackets + # and periods, so sanitize + + alpha_name = re.sub('[_\[\]\.]+', '_', cfg.name) + alpha_name = re.sub('_+$', '', alpha_name) + name = "%s_%s" % (cls.__name__, alpha_name) subcls = type( name, (cls, ), { - "__only_on__": ("%s+%s" % (cfg.db.name, cfg.db.driver)), + "_sa_orig_cls_name": orig_name, + "__only_on_config__": cfg } ) setattr(module, name, subcls) @@ -441,6 +465,12 @@ def _restore_engine(): config._current.reset(testing) +def final_process_cleanup(): + engines.testing_reaper._stop_test_ctx_aggressive() + assertions.global_cleanup_assertions() + _restore_engine() + + def _setup_engine(cls): if getattr(cls, '__engine_options__', None): eng = engines.testing_engine(options=cls.__engine_options__) @@ -451,11 +481,8 @@ def before_test(test, test_module_name, test_class, test_name): # like a nose id, e.g.: # "test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause" - name = test_class.__name__ - suffix = "_%s_%s" % (config.db.name, config.db.driver) - if name.endswith(suffix): - name = name[0:-(len(suffix))] + name = getattr(test_class, '_sa_orig_cls_name', test_class.__name__) id_ = "%s.%s.%s" % (test_module_name, name, test_name) @@ -481,6 +508,9 @@ def _possible_configs_for_cls(cls, reasons=None): if not spec(config_obj): all_configs.remove(config_obj) + if getattr(cls, '__only_on_config__', None): + all_configs.intersection_update([cls.__only_on_config__]) + if hasattr(cls, '__requires__'): requirements = config.requirements for config_obj in list(all_configs): diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index 67ccb35508..377b4643c8 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -42,10 +42,6 @@ def pytest_configure(config): plugin_base.configure_follower( config.slaveinput["follower_ident"] ) - - if config.option.write_idents: - with open(config.option.write_idents, "a") as file_: - file_.write(config.slaveinput["follower_ident"] + "\n") else: if config.option.write_idents and \ os.path.exists(config.option.write_idents): @@ -62,6 +58,11 @@ def pytest_configure(config): def pytest_sessionstart(session): plugin_base.post_begin() + +def pytest_sessionfinish(session): + plugin_base.final_process_cleanup() + + if has_xdist: import uuid diff --git a/lib/sqlalchemy/testing/provision.py b/lib/sqlalchemy/testing/provision.py index bb50a7ca7a..95fa2f8f75 100644 --- a/lib/sqlalchemy/testing/provision.py +++ b/lib/sqlalchemy/testing/provision.py @@ -1,11 +1,13 @@ from sqlalchemy.engine import url as sa_url +from sqlalchemy import create_engine from sqlalchemy import text from sqlalchemy import exc from sqlalchemy.util import compat from . import config, engines +import collections +import os import time import logging -import os log = logging.getLogger(__name__) FOLLOWER_IDENT = None @@ -41,6 +43,7 @@ class register(object): def create_follower_db(follower_ident): for cfg in _configs_for_db_operation(): + log.info("CREATE database %s, URI %r", follower_ident, cfg.db.url) _create_db(cfg, cfg.db, follower_ident) @@ -57,6 +60,7 @@ def setup_config(db_url, options, file_config, follower_ident): eng = engines.testing_engine(db_url, db_opts) _post_configure_engine(db_url, eng, follower_ident) eng.connect().close() + cfg = config.Config.register(eng, db_opts, options, file_config) if follower_ident: _configure_follower(cfg, follower_ident) @@ -65,6 +69,7 @@ def setup_config(db_url, options, file_config, follower_ident): def drop_follower_db(follower_ident): for cfg in _configs_for_db_operation(): + log.info("DROP database %s, URI %r", follower_ident, cfg.db.url) _drop_db(cfg, cfg.db, follower_ident) @@ -162,18 +167,21 @@ def _pg_create_db(cfg, eng, ident): except Exception: pass currentdb = conn.scalar("select current_database()") - for attempt in range(3): + for attempt in range(10): try: conn.execute( "CREATE DATABASE %s TEMPLATE %s" % (ident, currentdb)) except exc.OperationalError as err: - if attempt != 2 and "accessed by other users" in str(err): - time.sleep(.2) - continue - else: - raise + if "accessed by other users" in str(err): + log.info( + "Waiting to create %s, URI %r, " + "template DB is in use sleeping for .5", + ident, eng.url) + time.sleep(.5) else: break + else: + raise err @_create_db.for_db("mysql") @@ -276,36 +284,49 @@ def _oracle_update_db_opts(db_url, db_opts): db_opts['_retry_on_12516'] = True -def reap_oracle_dbs(eng, idents_file): +def reap_oracle_dbs(idents_file): log.info("Reaping Oracle dbs...") - with eng.connect() as conn: - with open(idents_file) as file_: - idents = set(line.strip() for line in file_) - - log.info("identifiers in file: %s", ", ".join(idents)) - - to_reap = conn.execute( - "select u.username from all_users u where username " - "like 'TEST_%' and not exists (select username " - "from v$session where username=u.username)") - all_names = set([username.lower() for (username, ) in to_reap]) - to_drop = set() - for name in all_names: - if name.endswith("_ts1") or name.endswith("_ts2"): - continue - elif name in idents: - to_drop.add(name) - if "%s_ts1" % name in all_names: - to_drop.add("%s_ts1" % name) - if "%s_ts2" % name in all_names: - to_drop.add("%s_ts2" % name) - - dropped = total = 0 - for total, username in enumerate(to_drop, 1): - if _ora_drop_ignore(conn, username): - dropped += 1 - log.info( - "Dropped %d out of %d stale databases detected", dropped, total) + + urls = collections.defaultdict(list) + with open(idents_file) as file_: + for line in file_: + line = line.strip() + db_name, db_url = line.split(" ") + urls[db_url].append(db_name) + + for url in urls: + if not url.startswith("oracle"): + continue + idents = urls[url] + log.info("db reaper connecting to %r", url) + eng = create_engine(url) + with eng.connect() as conn: + + log.info("identifiers in file: %s", ", ".join(idents)) + + to_reap = conn.execute( + "select u.username from all_users u where username " + "like 'TEST_%' and not exists (select username " + "from v$session where username=u.username)") + all_names = {username.lower() for (username, ) in to_reap} + to_drop = set() + for name in all_names: + if name.endswith("_ts1") or name.endswith("_ts2"): + continue + elif name in idents: + to_drop.add(name) + if "%s_ts1" % name in all_names: + to_drop.add("%s_ts1" % name) + if "%s_ts2" % name in all_names: + to_drop.add("%s_ts2" % name) + + dropped = total = 0 + for total, username in enumerate(to_drop, 1): + if _ora_drop_ignore(conn, username): + dropped += 1 + log.info( + "Dropped %d out of %d stale databases detected", + dropped, total) @_follower_url_from_main.for_db("oracle") diff --git a/reap_oracle_dbs.py b/reap_oracle_dbs.py index 4762faac55..29d2274643 100644 --- a/reap_oracle_dbs.py +++ b/reap_oracle_dbs.py @@ -6,8 +6,6 @@ TCP connection even if close() is called, which prevents the provisioning system from dropping a database in-process. """ -from sqlalchemy.testing.plugin import plugin_base -from sqlalchemy.testing import engines from sqlalchemy.testing import provision import logging import sys @@ -15,11 +13,6 @@ import sys logging.basicConfig() logging.getLogger(provision.__name__).setLevel(logging.INFO) -plugin_base.read_config() -oracle = plugin_base.file_config.get('db', 'oracle') -from sqlalchemy.testing import provision - -engine = engines.testing_engine(oracle, {}) -provision.reap_oracle_dbs(engine, sys.argv[1]) +provision.reap_oracle_dbs(sys.argv[1]) diff --git a/test/requirements.py b/test/requirements.py index d69692ffad..6833e5e9b5 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -882,8 +882,8 @@ class DefaultRequirements(SuiteRequirements): def check(config): if not against(config, 'mysql'): return False - row = config.db.execute("show variables like 'sql_mode'").first() - return not row or "STRICT" not in row[1] + row = config.db.execute("show variables like 'sql_mode'").first() + return not row or "STRICT" not in row[1] return only_if(check) diff --git a/tox_1_1.ini b/tox_1_1.ini index 9c4b6881e0..503039f78f 100644 --- a/tox_1_1.ini +++ b/tox_1_1.ini @@ -45,7 +45,7 @@ whitelist_externals=sh setenv= PYTHONPATH= PYTHONNOUSERSITE=1 - BASECOMMAND=python -m pytest + BASECOMMAND=python -m pytest --log-info=sqlalchemy.testing WORKERS={env:TOX_WORKERS:-n4} oracle: WORKERS={env:TOX_WORKERS:-n2}