From: Mike Bayer Date: Sun, 23 Feb 2025 16:20:18 +0000 (-0500) Subject: re-support mysql-connector python X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b056dd2c5ab71ce4143a95cd0fdd4a4190de19e6;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git re-support mysql-connector python Support has been re-added for the MySQL-Connector/Python DBAPI using the ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against modern MySQL versions as well as MariaDB versions (in the latter case it's required to pass charset/collation explicitly). Note however that server side cursor support is disabled due to unresolved issues with this driver. References: #12332 Change-Id: I81279478196e830d3c0d5f24ecb3fe2dc18d4ca6 --- diff --git a/doc/build/changelog/unreleased_20/12332.rst b/doc/build/changelog/unreleased_20/12332.rst new file mode 100644 index 0000000000..a6c1d4e2fb --- /dev/null +++ b/doc/build/changelog/unreleased_20/12332.rst @@ -0,0 +1,10 @@ +.. change:: + :tags: bug, mysql + :tickets: 12332 + + Support has been re-added for the MySQL-Connector/Python DBAPI using the + ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against + modern MySQL versions as well as MariaDB versions (in the latter case it's + required to pass charset/collation explicitly). Note however that + server side cursor support is disabled due to unresolved issues with this + driver. diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index df4d93c481..fd60d7ba65 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -2538,6 +2538,10 @@ class MySQLDialect(default.DefaultDialect): # allow for the "true" and "false" keywords, however supports_native_boolean = False + # support for BIT type; mysqlconnector coerces result values automatically, + # all other MySQL DBAPIs require a conversion routine + supports_native_bit = False + # identifiers are 64, however aliases can be 255... max_identifier_length = 255 max_index_name_length = 64 @@ -2739,10 +2743,12 @@ class MySQLDialect(default.DefaultDialect): % (".".join(map(str, server_version_info)),) ) if is_mariadb: - self.preparer = MariaDBIdentifierPreparer - # this would have been set by the default dialect already, - # so set it again - self.identifier_preparer = self.preparer(self) + + if not issubclass(self.preparer, MariaDBIdentifierPreparer): + self.preparer = MariaDBIdentifierPreparer + # this would have been set by the default dialect already, + # so set it again + self.identifier_preparer = self.preparer(self) # this will be updated on first connect in initialize() # if using older mariadb version diff --git a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py index a3ae490b5e..71ac58601c 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py +++ b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py @@ -14,25 +14,51 @@ r""" :connectstring: mysql+mysqlconnector://:@[:]/ :url: https://pypi.org/project/mysql-connector-python/ -.. note:: +Driver Status +------------- + +MySQL Connector/Python is supported as of SQLAlchemy 2.0.39 to the +degree which the driver is functional. There are still ongoing issues +with features such as server side cursors which remain disabled until +upstream issues are repaired. + +.. versionchanged:: 2.0.39 + + The MySQL Connector/Python dialect has been updated to support the + latest version of this DBAPI. Previously, MySQL Connector/Python + was not fully supported. + +Connecting to MariaDB with MySQL Connector/Python +-------------------------------------------------- + +MySQL Connector/Python may attempt to pass an incompatible collation to the +database when connecting to MariaDB. Experimentation has shown that using +``?charset=utf8mb4&collation=utfmb4_general_ci`` or similar MariaDB-compatible +charset/collation will allow connectivity. - The MySQL Connector/Python DBAPI has had many issues since its release, - some of which may remain unresolved, and the mysqlconnector dialect is - **not tested as part of SQLAlchemy's continuous integration**. - The recommended MySQL dialects are mysqlclient and PyMySQL. """ # noqa import re from .base import BIT +from .base import MariaDBIdentifierPreparer from .base import MySQLCompiler from .base import MySQLDialect +from .base import MySQLExecutionContext from .base import MySQLIdentifierPreparer from .mariadb import MariaDBDialect from ... import util +class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext): + def create_server_side_cursor(self): + return self._dbapi_connection.cursor(buffered=False) + + def create_default_cursor(self): + return self._dbapi_connection.cursor(buffered=True) + + class MySQLCompiler_mysqlconnector(MySQLCompiler): def visit_mod_binary(self, binary, operator, **kw): return ( @@ -42,7 +68,7 @@ class MySQLCompiler_mysqlconnector(MySQLCompiler): ) -class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer): +class IdentifierPreparerCommon_mysqlconnector: @property def _double_percents(self): return False @@ -56,6 +82,18 @@ class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer): return value +class MySQLIdentifierPreparer_mysqlconnector( + IdentifierPreparerCommon_mysqlconnector, MySQLIdentifierPreparer +): + pass + + +class MariaDBIdentifierPreparer_mysqlconnector( + IdentifierPreparerCommon_mysqlconnector, MariaDBIdentifierPreparer +): + pass + + class _myconnpyBIT(BIT): def result_processor(self, dialect, coltype): """MySQL-connector already converts mysql bits, so.""" @@ -72,9 +110,16 @@ class MySQLDialect_mysqlconnector(MySQLDialect): supports_native_decimal = True + supports_native_bit = True + + # not until https://bugs.mysql.com/bug.php?id=117548 + supports_server_side_cursors = False + default_paramstyle = "format" statement_compiler = MySQLCompiler_mysqlconnector + execution_ctx_cls = MySQLExecutionContext_mysqlconnector + preparer = MySQLIdentifierPreparer_mysqlconnector colspecs = util.update_copy(MySQLDialect.colspecs, {BIT: _myconnpyBIT}) @@ -112,9 +157,13 @@ class MySQLDialect_mysqlconnector(MySQLDialect): util.coerce_kw_type(opts, "use_pure", bool) util.coerce_kw_type(opts, "use_unicode", bool) - # unfortunately, MySQL/connector python refuses to release a - # cursor without reading fully, so non-buffered isn't an option - opts.setdefault("buffered", True) + # note that "buffered" is set to False by default in MySQL/connector + # python. If you set it to True, then there is no way to get a server + # side cursor because the logic is written to disallow that. + + # leaving this at True until + # https://bugs.mysql.com/bug.php?id=117548 can be fixed + opts["buffered"] = True # FOUND_ROWS must be set in ClientFlag to enable # supports_sane_rowcount. @@ -129,6 +178,7 @@ class MySQLDialect_mysqlconnector(MySQLDialect): opts["client_flags"] = client_flags except Exception: pass + return [[], opts] @util.memoized_property @@ -146,7 +196,11 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def is_disconnect(self, e, connection, cursor): errnos = (2006, 2013, 2014, 2045, 2055, 2048) - exceptions = (self.dbapi.OperationalError, self.dbapi.InterfaceError) + exceptions = ( + self.dbapi.OperationalError, + self.dbapi.InterfaceError, + self.dbapi.ProgrammingError, + ) if isinstance(e, exceptions): return ( e.errno in errnos @@ -162,20 +216,21 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def _compat_fetchone(self, rp, charset=None): return rp.fetchone() - _isolation_lookup = { - "SERIALIZABLE", - "READ UNCOMMITTED", - "READ COMMITTED", - "REPEATABLE READ", - "AUTOCOMMIT", - } + def get_isolation_level_values(self, dbapi_connection): + return ( + "SERIALIZABLE", + "READ UNCOMMITTED", + "READ COMMITTED", + "REPEATABLE READ", + "AUTOCOMMIT", + ) - def _set_isolation_level(self, connection, level): + def set_isolation_level(self, connection, level): if level == "AUTOCOMMIT": connection.autocommit = True else: connection.autocommit = False - super()._set_isolation_level(connection, level) + super().set_isolation_level(connection, level) class MariaDBDialect_mysqlconnector( @@ -183,6 +238,7 @@ class MariaDBDialect_mysqlconnector( ): supports_statement_cache = True _allows_uuid_binds = False + preparer = MariaDBIdentifierPreparer_mysqlconnector dialect = MySQLDialect_mysqlconnector diff --git a/lib/sqlalchemy/dialects/mysql/provision.py b/lib/sqlalchemy/dialects/mysql/provision.py index 7807af4097..46070848cb 100644 --- a/lib/sqlalchemy/dialects/mysql/provision.py +++ b/lib/sqlalchemy/dialects/mysql/provision.py @@ -42,6 +42,10 @@ def generate_driver_url(url, driver, query_str): if driver == "mariadbconnector": new_url = new_url.difference_update_query(["charset"]) + elif driver == "mysqlconnector": + new_url = new_url.update_query_pairs( + [("collation", "utf8mb4_general_ci")] + ) try: new_url.get_dialect() diff --git a/lib/sqlalchemy/dialects/mysql/types.py b/lib/sqlalchemy/dialects/mysql/types.py index d89d3776ea..015d51a105 100644 --- a/lib/sqlalchemy/dialects/mysql/types.py +++ b/lib/sqlalchemy/dialects/mysql/types.py @@ -380,12 +380,11 @@ class BIT(sqltypes.TypeEngine): self.length = length def result_processor(self, dialect, coltype): - """Convert a MySQL's 64 bit, variable length binary string to a long. + """Convert a MySQL's 64 bit, variable length binary string to a + long.""" - TODO: this is MySQL-db, pyodbc specific. OurSQL and mysqlconnector - already do this, so this logic should be moved to those dialects. - - """ + if dialect.supports_native_bit: + return None def process(value): if value is not None: diff --git a/lib/sqlalchemy/testing/suite/test_results.py b/lib/sqlalchemy/testing/suite/test_results.py index f22fd5ae13..4a71cb64fc 100644 --- a/lib/sqlalchemy/testing/suite/test_results.py +++ b/lib/sqlalchemy/testing/suite/test_results.py @@ -424,6 +424,8 @@ class ServerSideCursorsTest( return isinstance(cursor, sscursor) elif self.engine.dialect.driver == "mariadbconnector": return not cursor.buffered + elif self.engine.dialect.driver == "mysqlconnector": + return "buffered" not in type(cursor).__name__.lower() elif self.engine.dialect.driver in ("asyncpg", "aiosqlite"): return cursor.server_side elif self.engine.dialect.driver == "pg8000": diff --git a/setup.cfg b/setup.cfg index 76e1f1825b..bdcb3fd8db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,6 +72,7 @@ aiomysql = mysql+aiomysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 asyncmy = mysql+asyncmy://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 mariadb = mariadb+mysqldb://scott:tiger@127.0.0.1:3306/test mariadb_connector = mariadb+mariadbconnector://scott:tiger@127.0.0.1:3306/test +mysql_connector = mariadb+mysqlconnector://scott:tiger@127.0.0.1:3306/test mssql = mssql+pyodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional mssql_async = mssql+aioodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional pymssql = mssql+pymssql://scott:tiger^5HHH@mssql2022:1433/test diff --git a/test/dialect/mysql/test_dialect.py b/test/dialect/mysql/test_dialect.py index 23dbd39957..0dcc079a22 100644 --- a/test/dialect/mysql/test_dialect.py +++ b/test/dialect/mysql/test_dialect.py @@ -306,15 +306,19 @@ class DialectTest(fixtures.TestBase): )[1] eq_(kw["buffered"], True) - kw = dialect.create_connect_args( - make_url("mysql+mysqlconnector://u:p@host/db?buffered=false") - )[1] - eq_(kw["buffered"], False) - - kw = dialect.create_connect_args( - make_url("mysql+mysqlconnector://u:p@host/db") - )[1] - eq_(kw["buffered"], True) + # this is turned off for now due to + # https://bugs.mysql.com/bug.php?id=117548 + if dialect.supports_server_side_cursors: + kw = dialect.create_connect_args( + make_url("mysql+mysqlconnector://u:p@host/db?buffered=false") + )[1] + eq_(kw["buffered"], False) + + kw = dialect.create_connect_args( + make_url("mysql+mysqlconnector://u:p@host/db") + )[1] + # defaults to False as of 2.0.39 + eq_(kw.get("buffered"), None) def test_mysqlconnector_raise_on_warnings_arg(self): from sqlalchemy.dialects.mysql import mysqlconnector diff --git a/test/dialect/mysql/test_for_update.py b/test/dialect/mysql/test_for_update.py index 0895a098d1..5c26d8eb6d 100644 --- a/test/dialect/mysql/test_for_update.py +++ b/test/dialect/mysql/test_for_update.py @@ -90,7 +90,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): # set x/y > 10 try: alt_trans.execute(update(A).values(x=15, y=19)) - except (exc.InternalError, exc.OperationalError) as err: + except ( + exc.InternalError, + exc.OperationalError, + exc.DatabaseError, + ) as err: assert "Lock wait timeout exceeded" in str(err) assert should_be_locked else: @@ -103,7 +107,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): # set x/y > 10 try: alt_trans.execute(update(B).values(x=15, y=19)) - except (exc.InternalError, exc.OperationalError) as err: + except ( + exc.InternalError, + exc.OperationalError, + exc.DatabaseError, + ) as err: assert "Lock wait timeout exceeded" in str(err) assert should_be_locked else: diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 309d0e9eba..ab1491fd69 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -560,7 +560,7 @@ class ExecuteTest(fixtures.TablesTest): "Older versions don't support cursor pickling, newer ones do", ) @testing.fails_on( - "mysql+mysqlconnector", + "+mysqlconnector", "Exception doesn't come back exactly the same from pickle", ) @testing.fails_on( diff --git a/test/requirements.py b/test/requirements.py index 69b56423df..92fadf45da 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -1012,7 +1012,12 @@ class DefaultRequirements(SuiteRequirements): @property def arraysize(self): - return skip_if("+pymssql", "DBAPI is missing this attribute") + return skip_if( + [ + no_support("+pymssql", "DBAPI is missing this attribute"), + no_support("+mysqlconnector", "DBAPI ignores this attribute"), + ] + ) @property def emulated_lastrowid(self): diff --git a/tox.ini b/tox.ini index 1a5eb720db..9fefea2097 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ extras= mysql: mysql mysql: pymysql mysql: mariadb_connector + mysql: mysql_connector oracle: oracle oracle: oracle_oracledb @@ -142,8 +143,8 @@ setenv= memusage: WORKERS={env:TOX_WORKERS:-n2} mysql: MYSQL={env:TOX_MYSQL:--db mysql} - mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector} - mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector} + mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector --dbdriver mysqlconnector} + mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector --dbdriver mysqlconnector} mssql: MSSQL={env:TOX_MSSQL:--db mssql}