From: Georg Richter Date: Thu, 20 Aug 2020 12:00:14 +0000 (-0400) Subject: MariaDB dialect implementation X-Git-Tag: rel_1_4_0b1~133^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=603f7d30f68ecd515b57ce902fb12bbbebdca453;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git MariaDB dialect implementation Fixes: #5459 Closes: #5515 Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/5515 Pull-request-sha: 760090b9067304cc65fece12fcf10b522afc4a2a Change-Id: I30e8fbc02b7b5329ca228cd39f6fb7cfd0e43092 --- diff --git a/README.unittests.rst b/README.unittests.rst index 14b1bbcafb..84026007da 100644 --- a/README.unittests.rst +++ b/README.unittests.rst @@ -85,6 +85,7 @@ a pre-set URL. These can be seen using --dbs:: Available --db options (use --dburi to override) default sqlite:///:memory: firebird firebird://sysdba:masterkey@localhost//Users/classic/foo.fdb + mariadb mariadb://scott:tiger@192.168.0.199:3307/test mssql mssql+pyodbc://scott:tiger^5HHH@mssql2017:1433/test?driver=ODBC+Driver+13+for+SQL+Server mssql_pymssql mssql+pymssql://scott:tiger@ms_2008 mysql mysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 @@ -260,6 +261,21 @@ intended for production use! # To stop the container. It will also remove it. docker stop mysql +**MariaDB configuration**:: + + # only needed if a local image of MariaDB is not already present + docker pull mariadb + + # create the container with the proper configuration for sqlalchemy + docker run --rm -e MYSQL_USER='scott' -e MYSQL_PASSWORD='tiger' -e MYSQL_DATABASE='test' -e MYSQL_ROOT_PASSWORD='password' -p 127.0.0.1:3306:3306 -d --name mariadb mariadb --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + + # configure the database + sleep 20 + docker exec -ti mariadb mysql -u root -ppassword -D test -w -e "GRANT ALL ON *.* TO scott@'%'; CREATE DATABASE test_schema CHARSET utf8mb4; CREATE DATABASE test_schema_2 CHARSET utf8mb4;" + + # To stop the container. It will also remove it. + docker stop mariadb + **MSSQL configuration**:: # only needed if a local image of mssql is not already present diff --git a/doc/build/changelog/unreleased_14/5459.rst b/doc/build/changelog/unreleased_14/5459.rst new file mode 100644 index 0000000000..c892943442 --- /dev/null +++ b/doc/build/changelog/unreleased_14/5459.rst @@ -0,0 +1,6 @@ +.. change:: + :tags: engine, change + :tickets: 5459 + + Added support for MariaDB Connector/Python to the mysql dialect. Original + pull request courtesy Georg Richter. \ No newline at end of file diff --git a/lib/sqlalchemy/dialects/mysql/__init__.py b/lib/sqlalchemy/dialects/mysql/__init__.py index 683d438777..9fdc96f6fb 100644 --- a/lib/sqlalchemy/dialects/mysql/__init__.py +++ b/lib/sqlalchemy/dialects/mysql/__init__.py @@ -7,6 +7,7 @@ from . import base # noqa from . import cymysql # noqa +from . import mariadbconnector # noqa from . import mysqlconnector # noqa from . import mysqldb # noqa from . import oursql # noqa diff --git a/lib/sqlalchemy/dialects/mysql/mariadbconnector.py b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py new file mode 100644 index 0000000000..aa28ffc67b --- /dev/null +++ b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py @@ -0,0 +1,231 @@ +# mysql/mariadbconnector.py +# Copyright (C) 2020 the SQLAlchemy authors and contributors +# +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" + +.. dialect:: mysql+mariadbconnector + :name: MariaDB Connector/Python + :dbapi: mariadb + :connectstring: mariadb+mariadbconnector://:@[:]/ + :url: https://pypi.org/project/mariadb/ + +Driver Status +------------- + +MariaDB Connector/Python enables Python programs to access MariaDB and MySQL +databases using an API which is compliant with the Python DB API 2.0 (PEP-249). +It is written in C and uses MariaDB Connector/C client library for client server +communication. + +Note that the default driver for a ``mariadb://`` connection URI continues to +be ``mysqldb``. ``mariadb+mariadbconnector://`` is required to use this driver. + +.. mariadb: https://github.com/mariadb-corporation/mariadb-connector-python + +""" # noqa +import re + +from .base import MySQLCompiler +from .base import MySQLDialect +from .base import MySQLExecutionContext +from .base import MySQLIdentifierPreparer +from ... import sql +from ... import util + +mariadb_cpy_minimum_version = (1, 0, 1) + + +class MySQLExecutionContext_mariadbconnector(MySQLExecutionContext): + pass + + +class MySQLCompiler_mariadbconnector(MySQLCompiler): + pass + + +class MySQLIdentifierPreparer_mariadbconnector(MySQLIdentifierPreparer): + pass + + +class MySQLDialect_mariadbconnector(MySQLDialect): + driver = "mariadbconnector" + + # set this to True at the module level to prevent the driver from running + # against a backend that server detects as MySQL. currently this appears to + # be unnecessary as MariaDB client libraries have always worked against + # MySQL databases. However, if this changes at some point, this can be + # adjusted, but PLEASE ADD A TEST in test/dialect/mysql/test_dialect.py if + # this change is made at some point to ensure the correct exception + # is raised at the correct point when running the driver against + # a MySQL backend. + # is_mariadb = True + + supports_unicode_statements = True + encoding = "utf8mb4" + convert_unicode = True + supports_sane_rowcount = True + supports_sane_multi_rowcount = True + supports_native_decimal = True + default_paramstyle = "qmark" + execution_ctx_cls = MySQLExecutionContext_mariadbconnector + statement_compiler = MySQLCompiler_mariadbconnector + preparer = MySQLIdentifierPreparer_mariadbconnector + + @util.memoized_property + def _dbapi_version(self): + if self.dbapi and hasattr(self.dbapi, "__version__"): + return tuple( + [ + int(x) + for x in re.findall( + r"(\d+)(?:[-\.]?|$)", self.dbapi.__version__ + ) + ] + ) + else: + return (99, 99, 99) + + def __init__(self, server_side_cursors=False, **kwargs): + super(MySQLDialect_mariadbconnector, self).__init__(**kwargs) + self.server_side_cursors = True + self.paramstyle = "qmark" + if self.dbapi is not None: + if self._dbapi_version < mariadb_cpy_minimum_version: + raise NotImplementedError( + "The minimum required version for MariaDB " + "Connector/Python is %s" + % ".".join(str(x) for x in mariadb_cpy_minimum_version) + ) + + @classmethod + def dbapi(cls): + return __import__("mariadb") + + def is_disconnect(self, e, connection, cursor): + if super(MySQLDialect_mariadbconnector, self).is_disconnect( + e, connection, cursor + ): + return True + elif isinstance(e, self.dbapi.Error): + str_e = str(e).lower() + return "not connected" in str_e or "isn't valid" in str_e + else: + return False + + def create_connect_args(self, url): + opts = url.translate_connect_args() + + int_params = [ + "connect_timeout", + "read_timeout", + "write_timeout", + "client_flag", + "port", + "pool_size", + ] + bool_params = [ + "local_infile", + "ssl_verify_cert", + "ssl", + "pool_reset_connection", + ] + + for key in int_params: + util.coerce_kw_type(opts, key, int) + for key in bool_params: + util.coerce_kw_type(opts, key, bool) + + # FOUND_ROWS must be set in CLIENT_FLAGS to enable + # supports_sane_rowcount. + client_flag = opts.get("client_flag", 0) + if self.dbapi is not None: + try: + CLIENT_FLAGS = __import__( + self.dbapi.__name__ + ".constants.CLIENT" + ).constants.CLIENT + client_flag |= CLIENT_FLAGS.FOUND_ROWS + except (AttributeError, ImportError): + self.supports_sane_rowcount = False + opts["client_flag"] = client_flag + return [[], opts] + + def _extract_error_code(self, exception): + try: + rc = exception.errno + except: + rc = -1 + return rc + + def _detect_charset(self, connection): + return "utf8mb4" + + _isolation_lookup = set( + [ + "SERIALIZABLE", + "READ UNCOMMITTED", + "READ COMMITTED", + "REPEATABLE READ", + "AUTOCOMMIT", + ] + ) + + def _set_isolation_level(self, connection, level): + if level == "AUTOCOMMIT": + connection.autocommit = True + else: + connection.autocommit = False + super(MySQLDialect_mariadbconnector, self)._set_isolation_level( + connection, level + ) + + def do_begin_twophase(self, connection, xid): + connection.execute( + sql.text("XA BEGIN :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + + def do_prepare_twophase(self, connection, xid): + connection.execute( + sql.text("XA END :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + connection.execute( + sql.text("XA PREPARE :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + + def do_rollback_twophase( + self, connection, xid, is_prepared=True, recover=False + ): + if not is_prepared: + connection.execute( + sql.text("XA END :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + connection.execute( + sql.text("XA ROLLBACK :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + + def do_commit_twophase( + self, connection, xid, is_prepared=True, recover=False + ): + if not is_prepared: + self.do_prepare_twophase(connection, xid) + connection.execute( + sql.text("XA COMMIT :xid").bindparams( + sql.bindparam("xid", xid, literal_execute=True) + ) + ) + + +dialect = MySQLDialect_mariadbconnector diff --git a/lib/sqlalchemy/dialects/mysql/provision.py b/lib/sqlalchemy/dialects/mysql/provision.py index a1d82222db..c1d83bbb76 100644 --- a/lib/sqlalchemy/dialects/mysql/provision.py +++ b/lib/sqlalchemy/dialects/mysql/provision.py @@ -10,6 +10,18 @@ from ...testing.provision import temp_table_keyword_args def generate_driver_url(url, driver, query_str): backend = url.get_backend_name() + # NOTE: at the moment, tests are running mariadbconnector + # against both mariadb and mysql backends. if we want this to be + # limited, do the decisionmaking here to reject a "mysql+mariadbconnector" + # URL. Optionally also re-enable the module level + # MySQLDialect_mariadbconnector.is_mysql flag as well, which must include + # a unit and/or functional test. + + # all the Jenkins tests have been running mysqlclient Python library + # built against mariadb client drivers for years against all MySQL / + # MariaDB versions going back to MySQL 5.6, currently they can talk + # to MySQL databases without problems. + if backend == "mysql": dialect_cls = url.get_dialect() if dialect_cls._is_mariadb_from_url(url): diff --git a/setup.cfg b/setup.cfg index 387f422efd..cb8e6930ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,10 @@ mssql_pyodbc = pyodbc mysql = mysqlclient>=1.4.0,<2;python_version<"3" mysqlclient>=1.4.0;python_version>="3" +mysql_connector = + mysqlconnector +mariadb_connector = + mariadb>=1.0.1;python_version>="3" oracle = cx_oracle>=7,<8;python_version<"3" cx_oracle>=7;python_version>="3" @@ -120,6 +124,7 @@ pg8000 = postgresql+pg8000://scott:tiger@127.0.0.1:5432/test postgresql_psycopg2cffi = postgresql+psycopg2cffi://scott:tiger@127.0.0.1:5432/test mysql = mysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 pymysql = mysql+pymysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 +mariadb = mariadb://scott:tiger@127.0.0.1:3306/test mssql = mssql+pyodbc://scott:tiger^5HHH@mssql2017:1433/test?driver=ODBC+Driver+13+for+SQL+Server mssql_pymssql = mssql+pymssql://scott:tiger@ms_2008 docker_mssql = mssql+pymssql://scott:tiger^5HHH@127.0.0.1:1433/test diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index b9dd920a99..aec05130f4 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -453,7 +453,7 @@ class ExecuteTest(fixtures.TablesTest): try: cursor = raw.cursor() cursor.execute("SELECTINCORRECT") - except testing.db.dialect.dbapi.DatabaseError as orig: + except testing.db.dialect.dbapi.Error as orig: # py3k has "orig" in local scope... the_orig = orig finally: @@ -1506,6 +1506,7 @@ class EngineEventsTest(fixtures.TestBase): with e1.connect() as conn: result = conn.exec_driver_sql(stmt) + eq_(result.scalar(), 1) ctx = result.context eq_( diff --git a/test/engine/test_reconnect.py b/test/engine/test_reconnect.py index b8a8621dfa..0ec8fd0bfa 100644 --- a/test/engine/test_reconnect.py +++ b/test/engine/test_reconnect.py @@ -1356,6 +1356,7 @@ class InvalidateDuringResultTest(fixtures.TestBase): ) @testing.fails_if( [ + "+mariadbconnector", "+mysqlconnector", "+mysqldb", "+cymysql", diff --git a/tox.ini b/tox.ini index ac95dc42c6..ab5428f724 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,12 @@ deps=pytest!=3.9.1,!=3.9.2 postgresql: .[postgresql_pg8000]; python_version >= '3' mysql: .[mysql] mysql: .[pymysql] + mysql: .[mariadb_connector]; python_version >= '3' + + # we should probably try to get mysql_connector back in the mix + # as well + # mysql: .[mysql_connector]; python_version >= '3' + oracle: .[oracle] mssql: .[mssql] @@ -71,6 +77,7 @@ setenv= mysql: MYSQL={env:TOX_MYSQL:--db mysql} mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql} + py3{,5,6,7,8,9,10,11}-mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector} mssql: MSSQL={env:TOX_MSSQL:--db mssql}