From: Mike Bayer Date: Wed, 12 Aug 2020 22:46:25 +0000 (-0400) Subject: Use importlib_metadata; add namespace for mariadb X-Git-Tag: rel_1_4_0b1~180 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cd03b8f0cecbf72ecd6c99c4d3a6338c8278b40d;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Use importlib_metadata; add namespace for mariadb The ``importlib_metadata`` library is used to scan for setuptools entrypoints rather than pkg_resources. as importlib_metadata is a small library that is included as of Python 3.8, the compatibility library is installed as a dependency for Python versions older than 3.8. Unfortunately setuptools "attr:" is broken because it tries to import the module; seems like this is fixed as part of https://github.com/pypa/setuptools/pull/1753 however this is too recent to rely upon for now. Added a new dialect token "mariadb" that may be used in place of "mysql" in the :func:`_sa.create_engine` URL. This will deliver a MariaDB dialect subclass of the MySQLDialect in use that forces the "is_mariadb" flag to True. The dialect will raise an error if a server version string that does not indicate MariaDB in use is received. This is useful for MariaDB-specific testing scenarios as well as to support applications that are hardcoding to MariaDB-only concepts. As MariaDB and MySQL featuresets and usage patterns continue to diverge, this pattern may become more prominent. Fixes: #5400 Fixes: #5496 Change-Id: I330815ebe572b6a9818377da56621397335fa702 --- diff --git a/doc/build/changelog/unreleased_14/5400.rst b/doc/build/changelog/unreleased_14/5400.rst new file mode 100644 index 0000000000..8583b951cf --- /dev/null +++ b/doc/build/changelog/unreleased_14/5400.rst @@ -0,0 +1,9 @@ +.. change:: + :tags: change, installation + :tickets: 5400 + + The ``importlib_metadata`` library is used to scan for setuptools + entrypoints rather than pkg_resources. as importlib_metadata is a small + library that is included as of Python 3.8, the compatibility library is + installed as a dependency for Python versions older than 3.8. + diff --git a/doc/build/changelog/unreleased_14/5496.rst b/doc/build/changelog/unreleased_14/5496.rst new file mode 100644 index 0000000000..018378275b --- /dev/null +++ b/doc/build/changelog/unreleased_14/5496.rst @@ -0,0 +1,14 @@ +.. change:: + :tags: usecase, mysql + :tickets: 5496 + + Added a new dialect token "mariadb" that may be used in place of "mysql" in + the :func:`_sa.create_engine` URL. This will deliver a MariaDB dialect + subclass of the MySQLDialect in use that forces the "is_mariadb" flag to + True. The dialect will raise an error if a server version string that does + not indicate MariaDB in use is received. This is useful for + MariaDB-specific testing scenarios as well as to support applications that + are hardcoding to MariaDB-only concepts. As MariaDB and MySQL featuresets + and usage patterns continue to diverge, this pattern may become more + prominent. + diff --git a/doc/build/dialects/mysql.rst b/doc/build/dialects/mysql.rst index 65f7256479..1f2236155b 100644 --- a/doc/build/dialects/mysql.rst +++ b/doc/build/dialects/mysql.rst @@ -1,7 +1,7 @@ .. _mysql_toplevel: -MySQL -===== +MySQL and MariaDB +================= .. automodule:: sqlalchemy.dialects.mysql.base diff --git a/lib/sqlalchemy/dialects/__init__.py b/lib/sqlalchemy/dialects/__init__.py index 86f567eb54..4a79608d9e 100644 --- a/lib/sqlalchemy/dialects/__init__.py +++ b/lib/sqlalchemy/dialects/__init__.py @@ -15,6 +15,7 @@ __all__ = ( "sybase", ) + from .. import util @@ -44,6 +45,15 @@ def _auto_fn(name): except ImportError: module = __import__("sqlalchemy.dialects.sybase").dialects module = getattr(module, dialect) + elif dialect == "mariadb": + # it's "OK" for us to hardcode here since _auto_fn is already + # hardcoded. if mysql / mariadb etc were third party dialects + # they would just publish all the entrypoints, which would actually + # look much nicer. + module = __import__( + "sqlalchemy.dialects.mysql.mariadb" + ).dialects.mysql.mariadb + return module.loader(driver) else: module = __import__("sqlalchemy.dialects.%s" % (dialect,)).dialects module = getattr(module, dialect) diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index eea86bf758..de75f4104b 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -8,19 +8,50 @@ r""" .. dialect:: mysql - :name: MySQL + :name: MySQL / MariaDB Supported Versions and Features ------------------------------- -SQLAlchemy supports MySQL starting with version 4.1 through modern releases. -However, no heroic measures are taken to work around major missing -SQL features - if your server version does not support sub-selects, for -example, they won't work in SQLAlchemy either. +SQLAlchemy supports MySQL starting with version 4.1 through modern releases, as +well as all modern versions of MariaDB. However, no heroic measures are taken +to work around major missing SQL features - if your server version does not +support sub-selects, for example, they won't work in SQLAlchemy either. See the official MySQL documentation for detailed information about features supported in any given server release. +MariaDB Support +~~~~~~~~~~~~~~~ + +The MariaDB variant of MySQL retains fundamental compatibility with MySQL's +protocols however the development of these two products continues to diverge. +Within the realm of SQLAlchemy, the two databases have a small number of +syntactical and behavioral differences that SQLAlchemy accommodates automatically. +To connect to a MariaDB database, no changes to the database URL are required:: + + + engine = create_engine("mysql+pymsql://user:pass@some_mariadb/dbname?charset=utf8mb4") + +Upon first connect, the SQLAlchemy dialect employs a +server version detection scheme that determines if the +backing database reports as MariaDB. Based on this flag, the dialect +can make different choices in those of areas where its behavior +must be different. + +The dialect also supports a "MariaDB-only" mode of connection, which may be +useful for the case where an application makes use of MariaDB-specific features +and is not compatible with a MySQL database. To use this mode of operation, +replace the "mysql" token in the above URL with "mariadb":: + + engine = create_engine("mariadb+pymsql://user:pass@some_mariadb/dbname?charset=utf8mb4") + +The above engine, upon first connect, will raise an error if the server version +detection detects that the backing database is not MariaDB. + +.. versionadded:: 1.4 Added "mariadb" dialect name supporting "MariaDB-only mode" + for the MySQL dialect. + .. _mysql_connection_timeouts: Connection Timeouts and Disconnects @@ -1943,7 +1974,7 @@ class MySQLDDLCompiler(compiler.DDLCompiler): qual = "INDEX " const = self.preparer.format_constraint(constraint) elif isinstance(constraint, sa_schema.CheckConstraint): - if self.dialect._is_mariadb: + if self.dialect.is_mariadb: qual = "CONSTRAINT " else: qual = "CHECK " @@ -2352,6 +2383,8 @@ class MySQLDialect(default.DefaultDialect): ischema_names = ischema_names preparer = MySQLIdentifierPreparer + is_mariadb = False + # default SQL compilation settings - # these are modified upon initialize(), # i.e. first connect @@ -2378,6 +2411,7 @@ class MySQLDialect(default.DefaultDialect): isolation_level=None, json_serializer=None, json_deserializer=None, + is_mariadb=None, **kwargs ): kwargs.pop("use_ansiquotes", None) # legacy @@ -2385,6 +2419,7 @@ class MySQLDialect(default.DefaultDialect): self.isolation_level = isolation_level self._json_serializer = json_serializer self._json_deserializer = json_deserializer + self._set_mariadb(is_mariadb, None) def on_connect(self): if self.isolation_level is not None: @@ -2473,7 +2508,25 @@ class MySQLDialect(default.DefaultDialect): version.extend(g for g in mariadb.groups() if g) else: version.append(n) - return tuple(version) + + server_version_info = tuple(version) + + self._set_mariadb( + server_version_info and "MariaDB" in server_version_info, val + ) + + return server_version_info + + def _set_mariadb(self, is_mariadb, server_version_info): + if is_mariadb is None: + return + + if not is_mariadb and self.is_mariadb: + raise exc.InvalidRequestError( + "MySQL version %s is not a MariaDB variant." + % (server_version_info,) + ) + self.is_mariadb = is_mariadb def do_commit(self, dbapi_connection): """Execute a COMMIT.""" @@ -2677,7 +2730,7 @@ class MySQLDialect(default.DefaultDialect): default.DefaultDialect.initialize(self, connection) self.supports_sequences = ( - self._is_mariadb and self.server_version_info >= (10, 3) + self.is_mariadb and self.server_version_info >= (10, 3) ) self.supports_for_update_of = ( @@ -2685,13 +2738,13 @@ class MySQLDialect(default.DefaultDialect): ) self._needs_correct_for_88718_96365 = ( - not self._is_mariadb and self.server_version_info >= (8,) + not self.is_mariadb and self.server_version_info >= (8,) ) self._warn_for_known_db_issues() def _warn_for_known_db_issues(self): - if self._is_mariadb: + if self.is_mariadb: mdb_version = self._mariadb_normalized_version_info if mdb_version > (10, 2) and mdb_version < (10, 2, 9): util.warn( @@ -2706,17 +2759,15 @@ class MySQLDialect(default.DefaultDialect): @property def _is_mariadb(self): - return ( - self.server_version_info and "MariaDB" in self.server_version_info - ) + return self.is_mariadb @property def _is_mysql(self): - return not self._is_mariadb + return not self.is_mariadb @property def _is_mariadb_102(self): - return self._is_mariadb and self._mariadb_normalized_version_info > ( + return self.is_mariadb and self._mariadb_normalized_version_info > ( 10, 2, ) @@ -2726,7 +2777,7 @@ class MySQLDialect(default.DefaultDialect): # MariaDB's wire-protocol prepends the server_version with # the string "5.5"; now that we use @@version we no longer see this. - if self._is_mariadb: + if self.is_mariadb: idx = self.server_version_info.index("MariaDB") return self.server_version_info[idx - 3 : idx] else: diff --git a/lib/sqlalchemy/dialects/mysql/mariadb.py b/lib/sqlalchemy/dialects/mysql/mariadb.py new file mode 100644 index 0000000000..73db9eb225 --- /dev/null +++ b/lib/sqlalchemy/dialects/mysql/mariadb.py @@ -0,0 +1,16 @@ +from .base import MySQLDialect + + +class MariaDBDialect(MySQLDialect): + is_mariadb = True + + +def loader(driver): + driver_mod = __import__( + "sqlalchemy.dialects.mysql.%s" % driver + ).dialects.mysql + driver_cls = getattr(driver_mod, driver).dialect + + return type( + "MariaDBDialect_%s" % driver, (MariaDBDialect, driver_cls,), {} + ) diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index b5e6b0538f..caa97f72b0 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -15,6 +15,7 @@ import platform import sys +py38 = sys.version_info >= (3, 8) py37 = sys.version_info >= (3, 7) py36 = sys.version_info >= (3, 6) py3k = sys.version_info >= (3, 0) @@ -90,6 +91,11 @@ def inspect_getfullargspec(func): ) +if py38: + from importlib import metadata as importlib_metadata +else: + import importlib_metadata # noqa + if py3k: import base64 import builtins diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 28b7aa4ccc..cec54542a9 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -290,12 +290,10 @@ class PluginLoader(object): self.impls[name] = loader return loader() - try: - import pkg_resources - except ImportError: - pass - else: - for impl in pkg_resources.iter_entry_points(self.group, name): + for impl in compat.importlib_metadata.entry_points().get( + self.group, () + ): + if impl.name == name: self.impls[name] = impl.load return impl.load() diff --git a/setup.cfg b/setup.cfg index d1b46bc532..9cbdbd838d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,9 @@ [metadata] name = SQLAlchemy -version = attr: sqlalchemy.__version__ +# version comes from setup.py; setuptools +# can't read the "attr:" here without importing +# until version 47.0.0 which is too recent + description = Database Abstraction Library long_description = file: README.rst long_description_content_type = text/x-rst @@ -34,6 +37,9 @@ packages = find: python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* package_dir = =lib +install_requires = + importlib-metadata;python_version<"3.8" + [options.extras_require] mssql = pyodbc diff --git a/setup.py b/setup.py index 25ea7e2a22..341b1159ac 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from distutils.errors import DistutilsExecError from distutils.errors import DistutilsPlatformError import os import platform +import re import sys from setuptools import Distribution as _Distribution @@ -116,6 +117,16 @@ def status_msgs(*msgs): print("*" * 75) +with open( + os.path.join(os.path.dirname(__file__), "lib", "sqlalchemy", "__init__.py") +) as v_file: + VERSION = ( + re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S) + .match(v_file.read()) + .group(1) + ) + + def run_setup(with_cext): kwargs = {} if with_cext: @@ -129,7 +140,7 @@ def run_setup(with_cext): kwargs["ext_modules"] = [] - setup(cmdclass=cmdclass, distclass=Distribution, **kwargs) + setup(version=VERSION, cmdclass=cmdclass, distclass=Distribution, **kwargs) if not cpython: diff --git a/test/dialect/mysql/test_compiler.py b/test/dialect/mysql/test_compiler.py index 09bdd80be2..2053318b6e 100644 --- a/test/dialect/mysql/test_compiler.py +++ b/test/dialect/mysql/test_compiler.py @@ -150,13 +150,11 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): constraint_name = "constraint" constraint = CheckConstraint("data IS NOT NULL", name=constraint_name) Table(table_name, m, Column("data", String(255)), constraint) - dialect = mysql.dialect() - dialect.server_version_info = (10, 1, 1, "MariaDB") self.assert_compile( schema.DropConstraint(constraint), "ALTER TABLE %s DROP CONSTRAINT `%s`" % (table_name, constraint_name), - dialect=dialect, + dialect="mariadb", ) def test_create_index_with_length_quoted(self): @@ -354,8 +352,6 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): self.assert_compile(expr, "concat('x', 'y')", literal_binds=True) def test_mariadb_for_update(self): - dialect = mysql.dialect() - dialect.server_version_info = (10, 1, 1, "MariaDB") table1 = table( "mytable", column("myid"), column("name"), column("description") @@ -366,7 +362,7 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): "SELECT mytable.myid, mytable.name, mytable.description " "FROM mytable WHERE mytable.myid = %s " "FOR UPDATE", - dialect=dialect, + dialect="mariadb", ) self.assert_compile( @@ -376,7 +372,7 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): "SELECT mytable.myid, mytable.name, mytable.description " "FROM mytable WHERE mytable.myid = %s " "FOR UPDATE", - dialect=dialect, + dialect="mariadb", ) def test_delete_extra_froms(self): diff --git a/test/dialect/mysql/test_dialect.py b/test/dialect/mysql/test_dialect.py index d968d994c2..41a4af6395 100644 --- a/test/dialect/mysql/test_dialect.py +++ b/test/dialect/mysql/test_dialect.py @@ -5,6 +5,7 @@ import datetime from sqlalchemy import bindparam from sqlalchemy import Column from sqlalchemy import DateTime +from sqlalchemy import exc from sqlalchemy import func from sqlalchemy import Integer from sqlalchemy import MetaData @@ -12,18 +13,88 @@ from sqlalchemy import Table from sqlalchemy import testing from sqlalchemy.dialects import mysql from sqlalchemy.engine.url import make_url +from sqlalchemy.testing import assert_raises_message from sqlalchemy.testing import engines from sqlalchemy.testing import eq_ from sqlalchemy.testing import expect_warnings from sqlalchemy.testing import fixtures +from sqlalchemy.testing import is_ from sqlalchemy.testing import mock from ...engine import test_execute -class DialectTest(fixtures.TestBase): +class BackendDialectTest(fixtures.TestBase): __backend__ = True __only_on__ = "mysql" + def test_no_show_variables(self): + from sqlalchemy.testing import mock + + engine = engines.testing_engine() + + def my_execute(self, statement, *args, **kw): + if statement.startswith("SHOW VARIABLES"): + statement = "SELECT 1 FROM DUAL WHERE 1=0" + return real_exec(self, statement, *args, **kw) + + real_exec = engine._connection_cls.exec_driver_sql + with mock.patch.object( + engine._connection_cls, "exec_driver_sql", my_execute + ): + with expect_warnings( + "Could not retrieve SQL_MODE; please ensure the " + "MySQL user has permissions to SHOW VARIABLES" + ): + engine.connect() + + def test_no_default_isolation_level(self): + from sqlalchemy.testing import mock + + engine = engines.testing_engine() + + real_isolation_level = testing.db.dialect.get_isolation_level + + def fake_isolation_level(connection): + connection = mock.Mock( + cursor=mock.Mock( + return_value=mock.Mock( + fetchone=mock.Mock(return_value=None) + ) + ) + ) + return real_isolation_level(connection) + + with mock.patch.object( + engine.dialect, "get_isolation_level", fake_isolation_level + ): + with expect_warnings( + "Could not retrieve transaction isolation level for MySQL " + "connection." + ): + engine.connect() + + def test_autocommit_isolation_level(self): + c = testing.db.connect().execution_options( + isolation_level="AUTOCOMMIT" + ) + assert c.exec_driver_sql("SELECT @@autocommit;").scalar() + + c = c.execution_options(isolation_level="READ COMMITTED") + assert not c.exec_driver_sql("SELECT @@autocommit;").scalar() + + def test_isolation_level(self): + values = [ + "READ UNCOMMITTED", + "READ COMMITTED", + "REPEATABLE READ", + "SERIALIZABLE", + ] + for value in values: + c = testing.db.connect().execution_options(isolation_level=value) + eq_(testing.db.dialect.get_isolation_level(c.connection), value) + + +class DialectTest(fixtures.TestBase): @testing.combinations( (None, "cONnection was kILLEd", "InternalError", "pymysql", True), (None, "cONnection aLREady closed", "InternalError", "pymysql", True), @@ -176,74 +247,31 @@ class DialectTest(fixtures.TestBase): conn = eng.connect() eq_(conn.dialect._connection_charset, enc) - def test_no_show_variables(self): - from sqlalchemy.testing import mock - - engine = engines.testing_engine() - def my_execute(self, statement, *args, **kw): - if statement.startswith("SHOW VARIABLES"): - statement = "SELECT 1 FROM DUAL WHERE 1=0" - return real_exec(self, statement, *args, **kw) +class ParseVersionTest(fixtures.TestBase): + def test_mariadb_madness(self): + mysql_dialect = make_url("mysql://").get_dialect()() - real_exec = engine._connection_cls.exec_driver_sql - with mock.patch.object( - engine._connection_cls, "exec_driver_sql", my_execute - ): - with expect_warnings( - "Could not retrieve SQL_MODE; please ensure the " - "MySQL user has permissions to SHOW VARIABLES" - ): - engine.connect() + is_(mysql_dialect.is_mariadb, False) - def test_no_default_isolation_level(self): - from sqlalchemy.testing import mock + mysql_dialect = make_url("mysql+pymysql://").get_dialect()() + is_(mysql_dialect.is_mariadb, False) - engine = engines.testing_engine() + mariadb_dialect = make_url("mariadb://").get_dialect()() - real_isolation_level = testing.db.dialect.get_isolation_level + is_(mariadb_dialect.is_mariadb, True) - def fake_isolation_level(connection): - connection = mock.Mock( - cursor=mock.Mock( - return_value=mock.Mock( - fetchone=mock.Mock(return_value=None) - ) - ) - ) - return real_isolation_level(connection) + mariadb_dialect = make_url("mariadb+pymysql://").get_dialect()() - with mock.patch.object( - engine.dialect, "get_isolation_level", fake_isolation_level - ): - with expect_warnings( - "Could not retrieve transaction isolation level for MySQL " - "connection." - ): - engine.connect() + is_(mariadb_dialect.is_mariadb, True) - def test_autocommit_isolation_level(self): - c = testing.db.connect().execution_options( - isolation_level="AUTOCOMMIT" + assert_raises_message( + exc.InvalidRequestError, + "MySQL version 5.7.20 is not a MariaDB variant.", + mariadb_dialect._parse_server_version, + "5.7.20", ) - assert c.exec_driver_sql("SELECT @@autocommit;").scalar() - - c = c.execution_options(isolation_level="READ COMMITTED") - assert not c.exec_driver_sql("SELECT @@autocommit;").scalar() - - def test_isolation_level(self): - values = [ - "READ UNCOMMITTED", - "READ COMMITTED", - "REPEATABLE READ", - "SERIALIZABLE", - ] - for value in values: - c = testing.db.connect().execution_options(isolation_level=value) - eq_(testing.db.dialect.get_isolation_level(c.connection), value) - -class ParseVersionTest(fixtures.TestBase): @testing.combinations( ((10, 2, 7), "10.2.7-MariaDB", (10, 2, 7, "MariaDB"), True), ( @@ -286,7 +314,7 @@ class ParseVersionTest(fixtures.TestBase): (True, (10, 2, 6, "MariaDB", 10, 2, "6+maria~stretch", "log")), ) def test_mariadb_check_warning(self, expect_, version): - dialect = mysql.dialect() + dialect = mysql.dialect(is_mariadb="MariaDB" in version) dialect.server_version_info = version if expect_: with expect_warnings( diff --git a/tox.ini b/tox.ini index ead699a706..0ce79d7a1a 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ usedevelop= deps=pytest!=3.9.1,!=3.9.2 pytest-xdist mock; python_version < '3.3' - + importlib_metadata; python_version < '3.8' postgresql: .[postgresql] mysql: .[mysql] mysql: .[pymysql]