From: Mike Bayer Date: Fri, 14 Aug 2020 16:07:14 +0000 (-0400) Subject: Provision on different drivers dynamically X-Git-Tag: rel_1_4_0b1~172^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3231e281efb2f36dd0dbd628efcd684175064386;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Provision on different drivers dynamically We want TOX_POSTGRESQL and similar to be the fixed variable that is configured from CI environment. These variables should refer to database servers but individual drivers like asyncpg mysqlconnector etc. should come from local tox.ini. add a new system to generate per-driver URLs from a simple list of hostname-based URLs delivered from CI environment. Change-Id: I4267b4a70742765388c7e7c4432c1da9d9adece2 --- diff --git a/lib/sqlalchemy/dialects/postgresql/provision.py b/lib/sqlalchemy/dialects/postgresql/provision.py index 6c6dc4be64..eb82a411eb 100644 --- a/lib/sqlalchemy/dialects/postgresql/provision.py +++ b/lib/sqlalchemy/dialects/postgresql/provision.py @@ -1,13 +1,30 @@ +import copy import time from ... import exc from ... import text from ...testing.provision import create_db from ...testing.provision import drop_db +from ...testing.provision import generate_driver_url from ...testing.provision import log from ...testing.provision import temp_table_keyword_args +@generate_driver_url.for_db("postgresql") +def generate_driver_url(url, driver): + new_url = copy.copy(url) + new_url.drivername = "postgresql+%s" % driver + if new_url.get_driver_name() == "asyncpg": + new_url.query = dict(new_url.query) + new_url.query["async_fallback"] = "true" + try: + new_url.get_dialect() + except exc.NoSuchModuleError: + return None + else: + return new_url + + @create_db.for_db("postgresql") def _pg_create_db(cfg, eng, ident): template_db = cfg.options.postgresql_templatedb diff --git a/lib/sqlalchemy/testing/exclusions.py b/lib/sqlalchemy/testing/exclusions.py index 1a23ebf416..6ec4381930 100644 --- a/lib/sqlalchemy/testing/exclusions.py +++ b/lib/sqlalchemy/testing/exclusions.py @@ -272,6 +272,9 @@ class SpecPredicate(Predicate): } def __call__(self, config): + if config is None: + return False + engine = config.db if "+" in self.db: diff --git a/lib/sqlalchemy/testing/plugin/plugin_base.py b/lib/sqlalchemy/testing/plugin/plugin_base.py index 49ff0f9757..b5f2a3e0b8 100644 --- a/lib/sqlalchemy/testing/plugin/plugin_base.py +++ b/lib/sqlalchemy/testing/plugin/plugin_base.py @@ -17,10 +17,14 @@ is pytest. from __future__ import absolute_import import abc +import logging import re import sys +log = logging.getLogger("sqlalchemy.testing.plugin_base") + + py3k = sys.version_info >= (3, 0) if py3k: @@ -90,6 +94,15 @@ def setup_options(make_option): dest="dburi", help="Database uri. Multiple OK, " "first one is run by default.", ) + make_option( + "--dbdriver", + action="append", + type="string", + dest="dbdriver", + help="Additional database drivers to include in tests. " + "These are linked to the existing database URLs by the " + "provisioning system.", + ) make_option( "--dropfirst", action="store_true", @@ -350,6 +363,7 @@ def _init_symbols(options, file_config): @post def _engine_uri(options, file_config): + from sqlalchemy.testing import config from sqlalchemy import testing from sqlalchemy.testing import provision @@ -359,6 +373,8 @@ def _engine_uri(options, file_config): else: db_urls = [] + extra_drivers = options.dbdriver or [] + if options.db: for db_token in options.db: for db in re.split(r"[,\s]+", db_token): @@ -374,7 +390,11 @@ def _engine_uri(options, file_config): db_urls.append(file_config.get("db", "default")) config._current = None - for db_url in db_urls: + + expanded_urls = list(provision.generate_db_urls(db_urls, extra_drivers)) + + for db_url in expanded_urls: + log.info("Adding database URL: %s", db_url) if options.write_idents and provision.FOLLOWER_IDENT: # != 'master': with open(options.write_idents, "a") as file_: @@ -596,7 +616,8 @@ def stop_test_class(cls): def _restore_engine(): - config._current.reset(testing) + if config._current: + config._current.reset(testing) def final_process_cleanup(): diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index 3df239afa9..1b2bbca23d 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -89,9 +89,9 @@ def pytest_addoption(parser): def pytest_configure(config): - if hasattr(config, "slaveinput"): - plugin_base.restore_important_follower_config(config.slaveinput) - plugin_base.configure_follower(config.slaveinput["follower_ident"]) + if hasattr(config, "workerinput"): + plugin_base.restore_important_follower_config(config.workerinput) + plugin_base.configure_follower(config.workerinput["follower_ident"]) else: if config.option.write_idents and os.path.exists( config.option.write_idents @@ -162,20 +162,20 @@ if has_xdist: import uuid def pytest_configure_node(node): - # the master for each node fills slaveinput dictionary + # the master for each node fills workerinput dictionary # which pytest-xdist will transfer to the subprocess - plugin_base.memoize_important_follower_config(node.slaveinput) + plugin_base.memoize_important_follower_config(node.workerinput) - node.slaveinput["follower_ident"] = "test_%s" % uuid.uuid4().hex[0:12] + node.workerinput["follower_ident"] = "test_%s" % uuid.uuid4().hex[0:12] from sqlalchemy.testing import provision - provision.create_follower_db(node.slaveinput["follower_ident"]) + provision.create_follower_db(node.workerinput["follower_ident"]) def pytest_testnodedown(node, error): from sqlalchemy.testing import provision - provision.drop_follower_db(node.slaveinput["follower_ident"]) + provision.drop_follower_db(node.workerinput["follower_ident"]) def pytest_collection_modifyitems(session, config, items): diff --git a/lib/sqlalchemy/testing/provision.py b/lib/sqlalchemy/testing/provision.py index 660a3bd24c..13a5ea078c 100644 --- a/lib/sqlalchemy/testing/provision.py +++ b/lib/sqlalchemy/testing/provision.py @@ -1,8 +1,10 @@ import collections +import copy import logging from . import config from . import engines +from .. import exc from ..engine import url as sa_url from ..util import compat @@ -73,6 +75,91 @@ def drop_follower_db(follower_ident): drop_db(cfg, cfg.db, follower_ident) +def generate_db_urls(db_urls, extra_drivers): + """Generate a set of URLs to test given configured URLs plus additional + driver names. + + Given:: + + --dburi postgresql://db1 \ + --dburi postgresql://db2 \ + --dburi postgresql://db2 \ + --dbdriver=psycopg2 --dbdriver=asyncpg + + Noting that the default postgresql driver is psycopg2. the output + would be:: + + postgresql+psycopg2://db1 + postgresql+asyncpg://db1?async_fallback=true + postgresql+psycopg2://db2 + postgresql+psycopg2://db3 + + That is, for the driver in a --dburi, we want to keep that and use that + driver for each URL it's part of . For a driver that is only + in --dbdrivers, we want to use it just once for one of the URLs. + for a driver that is both coming from --dburi as well as --dbdrivers, + we want to keep it in that dburi. + + + """ + urls = set() + + backend_to_driver_we_already_have = collections.defaultdict(set) + + urls_plus_dialects = [ + (url_obj, url_obj.get_dialect()) + for url_obj in [sa_url.make_url(db_url) for db_url in db_urls] + ] + + for url_obj, dialect in urls_plus_dialects: + backend_to_driver_we_already_have[dialect.name].add(dialect.driver) + + backend_to_driver_we_need = {} + + for url_obj, dialect in urls_plus_dialects: + backend = dialect.name + dialect.load_provisioning() + + if backend not in backend_to_driver_we_need: + backend_to_driver_we_need[backend] = extra_per_backend = set( + extra_drivers + ).difference(backend_to_driver_we_already_have[backend]) + else: + extra_per_backend = backend_to_driver_we_need[backend] + + for driver_url in _generate_driver_urls(url_obj, extra_per_backend): + if driver_url in urls: + continue + urls.add(driver_url) + yield driver_url + + +def _generate_driver_urls(url, extra_drivers): + main_driver = url.get_driver_name() + extra_drivers.discard(main_driver) + + yield str(url) + + for drv in list(extra_drivers): + new_url = generate_driver_url(url, drv) + if new_url: + extra_drivers.remove(drv) + yield str(new_url) + + +@register.init +def generate_driver_url(url, driver): + backend = url.get_backend_name() + new_url = copy.copy(url) + new_url.drivername = "%s+%s" % (backend, driver) + try: + new_url.get_dialect() + except exc.NoSuchModuleError: + return None + else: + return new_url + + def _configs_for_db_operation(): hosts = set() diff --git a/tox.ini b/tox.ini index f230e5be12..92fe031724 100644 --- a/tox.ini +++ b/tox.ini @@ -52,31 +52,40 @@ setenv= BASECOMMAND=python -m pytest --log-info=sqlalchemy.testing WORKERS={env:TOX_WORKERS:-n4 --max-worker-restart=5} - oracle: WORKERS={env:TOX_WORKERS:-n2 --max-worker-restart=5} + nocext: DISABLE_SQLALCHEMY_CEXT=1 cext: REQUIRE_SQLALCHEMY_CEXT=1 cov: COVERAGE={[testenv]cov_args} + backendonly: BACKENDONLY=--backend-only + memusage: MEMUSAGE='-k test_memusage' + + oracle: WORKERS={env:TOX_WORKERS:-n2 --max-worker-restart=5} + oracle: ORACLE={env:TOX_ORACLE:--db oracle} + sqlite: SQLITE={env:TOX_SQLITE:--db sqlite} sqlite_file: SQLITE={env:TOX_SQLITE_FILE:--db sqlite_file} - postgresql: POSTGRESQL={env:TOX_POSTGRESQL_MASTER:--db postgresql} - mysql: MYSQL={env:TOX_MYSQL:--db mysql --db pymysql} - oracle: ORACLE={env:TOX_ORACLE:--db oracle} + + postgresql: POSTGRESQL={env:TOX_POSTGRESQL:--db postgresql} + postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver asyncpg} + + mysql: MYSQL={env:TOX_MYSQL:--db mysql} + mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql} + mssql: MSSQL={env:TOX_MSSQL:--db mssql} + oracle,mssql,sqlite_file: IDENTS=--write-idents db_idents.txt oracle,mssql,sqlite_file: MEMUSAGE=--nomemory - backendonly: BACKENDONLY=--backend-only - memusage: MEMUSAGE='-k test_memusage' # tox as of 2.0 blocks all environment variables from the # outside, unless they are here (or in TOX_TESTENV_PASSENV, # wildcards OK). Need at least these -passenv=ORACLE_HOME NLS_LANG TOX_POSTGRESQL_MASTER TOX_MYSQL TOX_ORACLE TOX_MSSQL TOX_SQLITE TOX_SQLITE_FILE TOX_WORKERS +passenv=ORACLE_HOME NLS_LANG TOX_POSTGRESQL TOX_MYSQL TOX_ORACLE TOX_MSSQL TOX_SQLITE TOX_SQLITE_FILE TOX_WORKERS EXTRA_PG_DRIVERS EXTRA_MYSQL_DRIVERS # for nocext, we rm *.so in lib in case we are doing usedevelop=True commands= cext: /bin/true nocext: sh -c "rm -f lib/sqlalchemy/*.so" - {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:MEMUSAGE:} {env:COVERAGE:} {posargs} + {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:EXTRA_PG_DRIVERS:} {env:MYSQL:} {env:EXTRA_MYSQL_DRIVERS:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:MEMUSAGE:} {env:COVERAGE:} {posargs} oracle,mssql,sqlite_file: python reap_dbs.py db_idents.txt # thanks to https://julien.danjou.info/the-best-flake8-extensions/