From: Mike Bayer Date: Sun, 23 Feb 2025 16:20:18 +0000 (-0500) Subject: re-support mysql-connector python X-Git-Tag: rel_2_0_40~20^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bdf4ef844caf622da96460d57b86f4cf41cdbd45;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git re-support mysql-connector python Support has been re-added for the MySQL-Connector/Python DBAPI using the ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against modern MySQL versions as well as MariaDB versions (in the latter case it's required to pass charset/collation explicitly). Note however that server side cursor support is disabled due to unresolved issues with this driver. note the 2.0 backport here necessarily needs to also backport some of 49ce245998 to handle the mariadb database working under mysql connector. References: #12332 Change-Id: I81279478196e830d3c0d5f24ecb3fe2dc18d4ca6 (cherry picked from commit b056dd2c5ab71ce4143a95cd0fdd4a4190de19e6) --- diff --git a/doc/build/changelog/unreleased_20/12332.rst b/doc/build/changelog/unreleased_20/12332.rst new file mode 100644 index 0000000000..a6c1d4e2fb --- /dev/null +++ b/doc/build/changelog/unreleased_20/12332.rst @@ -0,0 +1,10 @@ +.. change:: + :tags: bug, mysql + :tickets: 12332 + + Support has been re-added for the MySQL-Connector/Python DBAPI using the + ``mysql+mysqlconnector://`` URL scheme. The DBAPI now works against + modern MySQL versions as well as MariaDB versions (in the latter case it's + required to pass charset/collation explicitly). Note however that + server side cursor support is disabled due to unresolved issues with this + driver. diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 4a52d1b67a..8bae6193b5 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -2520,6 +2520,10 @@ class MySQLDialect(default.DefaultDialect): # allow for the "true" and "false" keywords, however supports_native_boolean = False + # support for BIT type; mysqlconnector coerces result values automatically, + # all other MySQL DBAPIs require a conversion routine + supports_native_bit = False + # identifiers are 64, however aliases can be 255... max_identifier_length = 255 max_index_name_length = 64 @@ -2721,10 +2725,12 @@ class MySQLDialect(default.DefaultDialect): % (".".join(map(str, server_version_info)),) ) if is_mariadb: - self.preparer = MariaDBIdentifierPreparer - # this would have been set by the default dialect already, - # so set it again - self.identifier_preparer = self.preparer(self) + + if not issubclass(self.preparer, MariaDBIdentifierPreparer): + self.preparer = MariaDBIdentifierPreparer + # this would have been set by the default dialect already, + # so set it again + self.identifier_preparer = self.preparer(self) # this will be updated on first connect in initialize() # if using older mariadb version diff --git a/lib/sqlalchemy/dialects/mysql/mariadb.py b/lib/sqlalchemy/dialects/mysql/mariadb.py index ac2cfbd1b0..b84dee37a7 100644 --- a/lib/sqlalchemy/dialects/mysql/mariadb.py +++ b/lib/sqlalchemy/dialects/mysql/mariadb.py @@ -46,16 +46,22 @@ class MariaDBDialect(MySQLDialect): def loader(driver): - driver_mod = __import__( + dialect_mod = __import__( "sqlalchemy.dialects.mysql.%s" % driver ).dialects.mysql - driver_cls = getattr(driver_mod, driver).dialect - - return type( - "MariaDBDialect_%s" % driver, - ( - MariaDBDialect, - driver_cls, - ), - {"supports_statement_cache": True}, - ) + + driver_mod = getattr(dialect_mod, driver) + if hasattr(driver_mod, "mariadb_dialect"): + driver_cls = driver_mod.mariadb_dialect + return driver_cls + else: + driver_cls = driver_mod.dialect + + return type( + "MariaDBDialect_%s" % driver, + ( + MariaDBDialect, + driver_cls, + ), + {"supports_statement_cache": True}, + ) diff --git a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py index e88f8fd71a..71ac58601c 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py +++ b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py @@ -14,24 +14,51 @@ r""" :connectstring: mysql+mysqlconnector://:@[:]/ :url: https://pypi.org/project/mysql-connector-python/ -.. note:: +Driver Status +------------- + +MySQL Connector/Python is supported as of SQLAlchemy 2.0.39 to the +degree which the driver is functional. There are still ongoing issues +with features such as server side cursors which remain disabled until +upstream issues are repaired. + +.. versionchanged:: 2.0.39 + + The MySQL Connector/Python dialect has been updated to support the + latest version of this DBAPI. Previously, MySQL Connector/Python + was not fully supported. + +Connecting to MariaDB with MySQL Connector/Python +-------------------------------------------------- + +MySQL Connector/Python may attempt to pass an incompatible collation to the +database when connecting to MariaDB. Experimentation has shown that using +``?charset=utf8mb4&collation=utfmb4_general_ci`` or similar MariaDB-compatible +charset/collation will allow connectivity. - The MySQL Connector/Python DBAPI has had many issues since its release, - some of which may remain unresolved, and the mysqlconnector dialect is - **not tested as part of SQLAlchemy's continuous integration**. - The recommended MySQL dialects are mysqlclient and PyMySQL. """ # noqa import re from .base import BIT +from .base import MariaDBIdentifierPreparer from .base import MySQLCompiler from .base import MySQLDialect +from .base import MySQLExecutionContext from .base import MySQLIdentifierPreparer +from .mariadb import MariaDBDialect from ... import util +class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext): + def create_server_side_cursor(self): + return self._dbapi_connection.cursor(buffered=False) + + def create_default_cursor(self): + return self._dbapi_connection.cursor(buffered=True) + + class MySQLCompiler_mysqlconnector(MySQLCompiler): def visit_mod_binary(self, binary, operator, **kw): return ( @@ -41,7 +68,7 @@ class MySQLCompiler_mysqlconnector(MySQLCompiler): ) -class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer): +class IdentifierPreparerCommon_mysqlconnector: @property def _double_percents(self): return False @@ -55,6 +82,18 @@ class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer): return value +class MySQLIdentifierPreparer_mysqlconnector( + IdentifierPreparerCommon_mysqlconnector, MySQLIdentifierPreparer +): + pass + + +class MariaDBIdentifierPreparer_mysqlconnector( + IdentifierPreparerCommon_mysqlconnector, MariaDBIdentifierPreparer +): + pass + + class _myconnpyBIT(BIT): def result_processor(self, dialect, coltype): """MySQL-connector already converts mysql bits, so.""" @@ -71,9 +110,16 @@ class MySQLDialect_mysqlconnector(MySQLDialect): supports_native_decimal = True + supports_native_bit = True + + # not until https://bugs.mysql.com/bug.php?id=117548 + supports_server_side_cursors = False + default_paramstyle = "format" statement_compiler = MySQLCompiler_mysqlconnector + execution_ctx_cls = MySQLExecutionContext_mysqlconnector + preparer = MySQLIdentifierPreparer_mysqlconnector colspecs = util.update_copy(MySQLDialect.colspecs, {BIT: _myconnpyBIT}) @@ -111,9 +157,13 @@ class MySQLDialect_mysqlconnector(MySQLDialect): util.coerce_kw_type(opts, "use_pure", bool) util.coerce_kw_type(opts, "use_unicode", bool) - # unfortunately, MySQL/connector python refuses to release a - # cursor without reading fully, so non-buffered isn't an option - opts.setdefault("buffered", True) + # note that "buffered" is set to False by default in MySQL/connector + # python. If you set it to True, then there is no way to get a server + # side cursor because the logic is written to disallow that. + + # leaving this at True until + # https://bugs.mysql.com/bug.php?id=117548 can be fixed + opts["buffered"] = True # FOUND_ROWS must be set in ClientFlag to enable # supports_sane_rowcount. @@ -128,6 +178,7 @@ class MySQLDialect_mysqlconnector(MySQLDialect): opts["client_flags"] = client_flags except Exception: pass + return [[], opts] @util.memoized_property @@ -145,7 +196,11 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def is_disconnect(self, e, connection, cursor): errnos = (2006, 2013, 2014, 2045, 2055, 2048) - exceptions = (self.dbapi.OperationalError, self.dbapi.InterfaceError) + exceptions = ( + self.dbapi.OperationalError, + self.dbapi.InterfaceError, + self.dbapi.ProgrammingError, + ) if isinstance(e, exceptions): return ( e.errno in errnos @@ -161,20 +216,30 @@ class MySQLDialect_mysqlconnector(MySQLDialect): def _compat_fetchone(self, rp, charset=None): return rp.fetchone() - _isolation_lookup = { - "SERIALIZABLE", - "READ UNCOMMITTED", - "READ COMMITTED", - "REPEATABLE READ", - "AUTOCOMMIT", - } + def get_isolation_level_values(self, dbapi_connection): + return ( + "SERIALIZABLE", + "READ UNCOMMITTED", + "READ COMMITTED", + "REPEATABLE READ", + "AUTOCOMMIT", + ) - def _set_isolation_level(self, connection, level): + def set_isolation_level(self, connection, level): if level == "AUTOCOMMIT": connection.autocommit = True else: connection.autocommit = False - super()._set_isolation_level(connection, level) + super().set_isolation_level(connection, level) + + +class MariaDBDialect_mysqlconnector( + MariaDBDialect, MySQLDialect_mysqlconnector +): + supports_statement_cache = True + _allows_uuid_binds = False + preparer = MariaDBIdentifierPreparer_mysqlconnector dialect = MySQLDialect_mysqlconnector +mariadb_dialect = MariaDBDialect_mysqlconnector diff --git a/lib/sqlalchemy/dialects/mysql/provision.py b/lib/sqlalchemy/dialects/mysql/provision.py index 7807af4097..46070848cb 100644 --- a/lib/sqlalchemy/dialects/mysql/provision.py +++ b/lib/sqlalchemy/dialects/mysql/provision.py @@ -42,6 +42,10 @@ def generate_driver_url(url, driver, query_str): if driver == "mariadbconnector": new_url = new_url.difference_update_query(["charset"]) + elif driver == "mysqlconnector": + new_url = new_url.update_query_pairs( + [("collation", "utf8mb4_general_ci")] + ) try: new_url.get_dialect() diff --git a/lib/sqlalchemy/dialects/mysql/types.py b/lib/sqlalchemy/dialects/mysql/types.py index 0c05aacb7c..ace6824a74 100644 --- a/lib/sqlalchemy/dialects/mysql/types.py +++ b/lib/sqlalchemy/dialects/mysql/types.py @@ -374,12 +374,11 @@ class BIT(sqltypes.TypeEngine): self.length = length def result_processor(self, dialect, coltype): - """Convert a MySQL's 64 bit, variable length binary string to a long. + """Convert a MySQL's 64 bit, variable length binary string to a + long.""" - TODO: this is MySQL-db, pyodbc specific. OurSQL and mysqlconnector - already do this, so this logic should be moved to those dialects. - - """ + if dialect.supports_native_bit: + return None def process(value): if value is not None: diff --git a/lib/sqlalchemy/testing/suite/test_results.py b/lib/sqlalchemy/testing/suite/test_results.py index a6179d8559..317195fd1e 100644 --- a/lib/sqlalchemy/testing/suite/test_results.py +++ b/lib/sqlalchemy/testing/suite/test_results.py @@ -268,6 +268,8 @@ class ServerSideCursorsTest( return isinstance(cursor, sscursor) elif self.engine.dialect.driver == "mariadbconnector": return not cursor.buffered + elif self.engine.dialect.driver == "mysqlconnector": + return "buffered" not in type(cursor).__name__.lower() elif self.engine.dialect.driver in ("asyncpg", "aiosqlite"): return cursor.server_side elif self.engine.dialect.driver == "pg8000": diff --git a/setup.cfg b/setup.cfg index bf9aedd8b6..21ab137425 100644 --- a/setup.cfg +++ b/setup.cfg @@ -179,6 +179,7 @@ asyncmy = mysql+asyncmy://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 asyncmy_fallback = mysql+asyncmy://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4&async_fallback=true mariadb = mariadb+mysqldb://scott:tiger@127.0.0.1:3306/test mariadb_connector = mariadb+mariadbconnector://scott:tiger@127.0.0.1:3306/test +mysql_connector = mariadb+mysqlconnector://scott:tiger@127.0.0.1:3306/test mssql = mssql+pyodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional mssql_async = mssql+aioodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional pymssql = mssql+pymssql://scott:tiger^5HHH@mssql2022:1433/test diff --git a/test/dialect/mysql/test_dialect.py b/test/dialect/mysql/test_dialect.py index cf74f17ad6..7e31c666f3 100644 --- a/test/dialect/mysql/test_dialect.py +++ b/test/dialect/mysql/test_dialect.py @@ -302,15 +302,19 @@ class DialectTest(fixtures.TestBase): )[1] eq_(kw["buffered"], True) - kw = dialect.create_connect_args( - make_url("mysql+mysqlconnector://u:p@host/db?buffered=false") - )[1] - eq_(kw["buffered"], False) - - kw = dialect.create_connect_args( - make_url("mysql+mysqlconnector://u:p@host/db") - )[1] - eq_(kw["buffered"], True) + # this is turned off for now due to + # https://bugs.mysql.com/bug.php?id=117548 + if dialect.supports_server_side_cursors: + kw = dialect.create_connect_args( + make_url("mysql+mysqlconnector://u:p@host/db?buffered=false") + )[1] + eq_(kw["buffered"], False) + + kw = dialect.create_connect_args( + make_url("mysql+mysqlconnector://u:p@host/db") + )[1] + # defaults to False as of 2.0.39 + eq_(kw.get("buffered"), None) def test_mysqlconnector_raise_on_warnings_arg(self): from sqlalchemy.dialects.mysql import mysqlconnector diff --git a/test/dialect/mysql/test_for_update.py b/test/dialect/mysql/test_for_update.py index 0895a098d1..5c26d8eb6d 100644 --- a/test/dialect/mysql/test_for_update.py +++ b/test/dialect/mysql/test_for_update.py @@ -90,7 +90,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): # set x/y > 10 try: alt_trans.execute(update(A).values(x=15, y=19)) - except (exc.InternalError, exc.OperationalError) as err: + except ( + exc.InternalError, + exc.OperationalError, + exc.DatabaseError, + ) as err: assert "Lock wait timeout exceeded" in str(err) assert should_be_locked else: @@ -103,7 +107,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): # set x/y > 10 try: alt_trans.execute(update(B).values(x=15, y=19)) - except (exc.InternalError, exc.OperationalError) as err: + except ( + exc.InternalError, + exc.OperationalError, + exc.DatabaseError, + ) as err: assert "Lock wait timeout exceeded" in str(err) assert should_be_locked else: diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py index 3291fa3047..28541ca33a 100644 --- a/test/engine/test_execute.py +++ b/test/engine/test_execute.py @@ -555,7 +555,7 @@ class ExecuteTest(fixtures.TablesTest): "Older versions don't support cursor pickling, newer ones do", ) @testing.fails_on( - "mysql+mysqlconnector", + "+mysqlconnector", "Exception doesn't come back exactly the same from pickle", ) @testing.fails_on( diff --git a/test/requirements.py b/test/requirements.py index 98a98cd74e..12c25ece1a 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -1012,7 +1012,12 @@ class DefaultRequirements(SuiteRequirements): @property def arraysize(self): - return skip_if("+pymssql", "DBAPI is missing this attribute") + return skip_if( + [ + no_support("+pymssql", "DBAPI is missing this attribute"), + no_support("+mysqlconnector", "DBAPI ignores this attribute"), + ] + ) @property def emulated_lastrowid(self): diff --git a/tox.ini b/tox.ini index ca7177b2ec..ae11373548 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ extras= mysql: mysql mysql: pymysql mysql: mariadb_connector + mysql: mysql_connector oracle: oracle oracle: oracle_oracledb @@ -145,8 +146,8 @@ setenv= memusage: WORKERS={env:TOX_WORKERS:-n2} mysql: MYSQL={env:TOX_MYSQL:--db mysql} - mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector} - mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector} + mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector --dbdriver mysqlconnector} + mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector --dbdriver mysqlconnector} mssql: MSSQL={env:TOX_MSSQL:--db mssql} mssql: EXTRA_MSSQL_DRIVERS={env:EXTRA_MSSQL_DRIVERS:--dbdriver pyodbc --dbdriver aioodbc --dbdriver pymssql}