]> git.ipfire.org Git - thirdparty/sqlalchemy/sqlalchemy.git/commitdiff
re-support mysql-connector python
authorMike Bayer <mike_mp@zzzcomputing.com>
Sun, 23 Feb 2025 16:20:18 +0000 (11:20 -0500)
committerMike Bayer <mike_mp@zzzcomputing.com>
Mon, 10 Mar 2025 18:04:21 +0000 (14:04 -0400)
Support has been re-added for the MySQL-Connector/Python DBAPI using the
``mysql+mysqlconnector://`` URL scheme.   The DBAPI now works against
modern MySQL versions as well as MariaDB versions (in the latter case it's
required to pass charset/collation explicitly).   Note however that
server side cursor support is disabled due to unresolved issues with this
driver.

References: #12332
Change-Id: I81279478196e830d3c0d5f24ecb3fe2dc18d4ca6

12 files changed:
doc/build/changelog/unreleased_20/12332.rst [new file with mode: 0644]
lib/sqlalchemy/dialects/mysql/base.py
lib/sqlalchemy/dialects/mysql/mysqlconnector.py
lib/sqlalchemy/dialects/mysql/provision.py
lib/sqlalchemy/dialects/mysql/types.py
lib/sqlalchemy/testing/suite/test_results.py
setup.cfg
test/dialect/mysql/test_dialect.py
test/dialect/mysql/test_for_update.py
test/engine/test_execute.py
test/requirements.py
tox.ini

diff --git a/doc/build/changelog/unreleased_20/12332.rst b/doc/build/changelog/unreleased_20/12332.rst
new file mode 100644 (file)
index 0000000..a6c1d4e
--- /dev/null
@@ -0,0 +1,10 @@
+.. change::
+    :tags: bug, mysql
+    :tickets: 12332
+
+    Support has been re-added for the MySQL-Connector/Python DBAPI using the
+    ``mysql+mysqlconnector://`` URL scheme.   The DBAPI now works against
+    modern MySQL versions as well as MariaDB versions (in the latter case it's
+    required to pass charset/collation explicitly).   Note however that
+    server side cursor support is disabled due to unresolved issues with this
+    driver.
index df4d93c4811d9291c0fc56d959e6bdcf3ca53ea0..fd60d7ba65c7072cc5115496c2fa63e64e46e6af 100644 (file)
@@ -2538,6 +2538,10 @@ class MySQLDialect(default.DefaultDialect):
     # allow for the "true" and "false" keywords, however
     supports_native_boolean = False
 
+    # support for BIT type; mysqlconnector coerces result values automatically,
+    # all other MySQL DBAPIs require a conversion routine
+    supports_native_bit = False
+
     # identifiers are 64, however aliases can be 255...
     max_identifier_length = 255
     max_index_name_length = 64
@@ -2739,10 +2743,12 @@ class MySQLDialect(default.DefaultDialect):
                 % (".".join(map(str, server_version_info)),)
             )
         if is_mariadb:
-            self.preparer = MariaDBIdentifierPreparer
-            # this would have been set by the default dialect already,
-            # so set it again
-            self.identifier_preparer = self.preparer(self)
+
+            if not issubclass(self.preparer, MariaDBIdentifierPreparer):
+                self.preparer = MariaDBIdentifierPreparer
+                # this would have been set by the default dialect already,
+                # so set it again
+                self.identifier_preparer = self.preparer(self)
 
             # this will be updated on first connect in initialize()
             # if using older mariadb version
index a3ae490b5ea79e5e14cc91cfd909c01f03b2d525..71ac58601c19e250342d0eb6ca04771f193a2563 100644 (file)
@@ -14,25 +14,51 @@ r"""
     :connectstring: mysql+mysqlconnector://<user>:<password>@<host>[:<port>]/<dbname>
     :url: https://pypi.org/project/mysql-connector-python/
 
-.. note::
+Driver Status
+-------------
+
+MySQL Connector/Python is supported as of SQLAlchemy 2.0.39 to the
+degree which the driver is functional.   There are still ongoing issues
+with features such as server side cursors which remain disabled until
+upstream issues are repaired.
+
+.. versionchanged:: 2.0.39
+
+    The MySQL Connector/Python dialect has been updated to support the
+    latest version of this DBAPI.   Previously, MySQL Connector/Python
+    was not fully supported.
+
+Connecting to MariaDB with MySQL Connector/Python
+--------------------------------------------------
+
+MySQL Connector/Python may attempt to pass an incompatible collation to the
+database when connecting to MariaDB.  Experimentation has shown that using
+``?charset=utf8mb4&collation=utfmb4_general_ci`` or similar MariaDB-compatible
+charset/collation will allow connectivity.
 
-    The MySQL Connector/Python DBAPI has had many issues since its release,
-    some of which may remain unresolved, and the mysqlconnector dialect is
-    **not tested as part of SQLAlchemy's continuous integration**.
-    The recommended MySQL dialects are mysqlclient and PyMySQL.
 
 """  # noqa
 
 import re
 
 from .base import BIT
+from .base import MariaDBIdentifierPreparer
 from .base import MySQLCompiler
 from .base import MySQLDialect
+from .base import MySQLExecutionContext
 from .base import MySQLIdentifierPreparer
 from .mariadb import MariaDBDialect
 from ... import util
 
 
+class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext):
+    def create_server_side_cursor(self):
+        return self._dbapi_connection.cursor(buffered=False)
+
+    def create_default_cursor(self):
+        return self._dbapi_connection.cursor(buffered=True)
+
+
 class MySQLCompiler_mysqlconnector(MySQLCompiler):
     def visit_mod_binary(self, binary, operator, **kw):
         return (
@@ -42,7 +68,7 @@ class MySQLCompiler_mysqlconnector(MySQLCompiler):
         )
 
 
-class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer):
+class IdentifierPreparerCommon_mysqlconnector:
     @property
     def _double_percents(self):
         return False
@@ -56,6 +82,18 @@ class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer):
         return value
 
 
+class MySQLIdentifierPreparer_mysqlconnector(
+    IdentifierPreparerCommon_mysqlconnector, MySQLIdentifierPreparer
+):
+    pass
+
+
+class MariaDBIdentifierPreparer_mysqlconnector(
+    IdentifierPreparerCommon_mysqlconnector, MariaDBIdentifierPreparer
+):
+    pass
+
+
 class _myconnpyBIT(BIT):
     def result_processor(self, dialect, coltype):
         """MySQL-connector already converts mysql bits, so."""
@@ -72,9 +110,16 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
 
     supports_native_decimal = True
 
+    supports_native_bit = True
+
+    # not until https://bugs.mysql.com/bug.php?id=117548
+    supports_server_side_cursors = False
+
     default_paramstyle = "format"
     statement_compiler = MySQLCompiler_mysqlconnector
 
+    execution_ctx_cls = MySQLExecutionContext_mysqlconnector
+
     preparer = MySQLIdentifierPreparer_mysqlconnector
 
     colspecs = util.update_copy(MySQLDialect.colspecs, {BIT: _myconnpyBIT})
@@ -112,9 +157,13 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
         util.coerce_kw_type(opts, "use_pure", bool)
         util.coerce_kw_type(opts, "use_unicode", bool)
 
-        # unfortunately, MySQL/connector python refuses to release a
-        # cursor without reading fully, so non-buffered isn't an option
-        opts.setdefault("buffered", True)
+        # note that "buffered" is set to False by default in MySQL/connector
+        # python.  If you set it to True, then there is no way to get a server
+        # side cursor because the logic is written to disallow that.
+
+        # leaving this at True until
+        # https://bugs.mysql.com/bug.php?id=117548 can be fixed
+        opts["buffered"] = True
 
         # FOUND_ROWS must be set in ClientFlag to enable
         # supports_sane_rowcount.
@@ -129,6 +178,7 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
                 opts["client_flags"] = client_flags
             except Exception:
                 pass
+
         return [[], opts]
 
     @util.memoized_property
@@ -146,7 +196,11 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
 
     def is_disconnect(self, e, connection, cursor):
         errnos = (2006, 2013, 2014, 2045, 2055, 2048)
-        exceptions = (self.dbapi.OperationalError, self.dbapi.InterfaceError)
+        exceptions = (
+            self.dbapi.OperationalError,
+            self.dbapi.InterfaceError,
+            self.dbapi.ProgrammingError,
+        )
         if isinstance(e, exceptions):
             return (
                 e.errno in errnos
@@ -162,20 +216,21 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
     def _compat_fetchone(self, rp, charset=None):
         return rp.fetchone()
 
-    _isolation_lookup = {
-        "SERIALIZABLE",
-        "READ UNCOMMITTED",
-        "READ COMMITTED",
-        "REPEATABLE READ",
-        "AUTOCOMMIT",
-    }
+    def get_isolation_level_values(self, dbapi_connection):
+        return (
+            "SERIALIZABLE",
+            "READ UNCOMMITTED",
+            "READ COMMITTED",
+            "REPEATABLE READ",
+            "AUTOCOMMIT",
+        )
 
-    def _set_isolation_level(self, connection, level):
+    def set_isolation_level(self, connection, level):
         if level == "AUTOCOMMIT":
             connection.autocommit = True
         else:
             connection.autocommit = False
-            super()._set_isolation_level(connection, level)
+            super().set_isolation_level(connection, level)
 
 
 class MariaDBDialect_mysqlconnector(
@@ -183,6 +238,7 @@ class MariaDBDialect_mysqlconnector(
 ):
     supports_statement_cache = True
     _allows_uuid_binds = False
+    preparer = MariaDBIdentifierPreparer_mysqlconnector
 
 
 dialect = MySQLDialect_mysqlconnector
index 7807af40975b2a9d836b5568b59c8cbfadcf3f7c..46070848cb11743e5874366e2498ff8afa869b42 100644 (file)
@@ -42,6 +42,10 @@ def generate_driver_url(url, driver, query_str):
 
     if driver == "mariadbconnector":
         new_url = new_url.difference_update_query(["charset"])
+    elif driver == "mysqlconnector":
+        new_url = new_url.update_query_pairs(
+            [("collation", "utf8mb4_general_ci")]
+        )
 
     try:
         new_url.get_dialect()
index d89d3776ea2056b129f60d4c6a47c33aaf70ee34..015d51a105816ff0e26250029830194a8e7cde92 100644 (file)
@@ -380,12 +380,11 @@ class BIT(sqltypes.TypeEngine):
         self.length = length
 
     def result_processor(self, dialect, coltype):
-        """Convert a MySQL's 64 bit, variable length binary string to a long.
+        """Convert a MySQL's 64 bit, variable length binary string to a
+        long."""
 
-        TODO: this is MySQL-db, pyodbc specific.  OurSQL and mysqlconnector
-        already do this, so this logic should be moved to those dialects.
-
-        """
+        if dialect.supports_native_bit:
+            return None
 
         def process(value):
             if value is not None:
index f22fd5ae13546b914ecbb86a13bddfe6b79eb3da..4a71cb64fc4961d0851a5c707a93fd1e7e03206d 100644 (file)
@@ -424,6 +424,8 @@ class ServerSideCursorsTest(
             return isinstance(cursor, sscursor)
         elif self.engine.dialect.driver == "mariadbconnector":
             return not cursor.buffered
+        elif self.engine.dialect.driver == "mysqlconnector":
+            return "buffered" not in type(cursor).__name__.lower()
         elif self.engine.dialect.driver in ("asyncpg", "aiosqlite"):
             return cursor.server_side
         elif self.engine.dialect.driver == "pg8000":
index 76e1f1825b066e0a19257dcf5e23192935998f58..bdcb3fd8db7d88db4be4b36b4f49e63d0ff556f2 100644 (file)
--- a/setup.cfg
+++ b/setup.cfg
@@ -72,6 +72,7 @@ aiomysql = mysql+aiomysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4
 asyncmy = mysql+asyncmy://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4
 mariadb = mariadb+mysqldb://scott:tiger@127.0.0.1:3306/test
 mariadb_connector = mariadb+mariadbconnector://scott:tiger@127.0.0.1:3306/test
+mysql_connector = mariadb+mysqlconnector://scott:tiger@127.0.0.1:3306/test
 mssql = mssql+pyodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional
 mssql_async = mssql+aioodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional
 pymssql = mssql+pymssql://scott:tiger^5HHH@mssql2022:1433/test
index 23dbd39957f8a19a804fe0f5a5386545bf914c8e..0dcc079a22bcd7e218badb48ae18c201def26054 100644 (file)
@@ -306,15 +306,19 @@ class DialectTest(fixtures.TestBase):
         )[1]
         eq_(kw["buffered"], True)
 
-        kw = dialect.create_connect_args(
-            make_url("mysql+mysqlconnector://u:p@host/db?buffered=false")
-        )[1]
-        eq_(kw["buffered"], False)
-
-        kw = dialect.create_connect_args(
-            make_url("mysql+mysqlconnector://u:p@host/db")
-        )[1]
-        eq_(kw["buffered"], True)
+        # this is turned off for now due to
+        # https://bugs.mysql.com/bug.php?id=117548
+        if dialect.supports_server_side_cursors:
+            kw = dialect.create_connect_args(
+                make_url("mysql+mysqlconnector://u:p@host/db?buffered=false")
+            )[1]
+            eq_(kw["buffered"], False)
+
+            kw = dialect.create_connect_args(
+                make_url("mysql+mysqlconnector://u:p@host/db")
+            )[1]
+            # defaults to False as of 2.0.39
+            eq_(kw.get("buffered"), None)
 
     def test_mysqlconnector_raise_on_warnings_arg(self):
         from sqlalchemy.dialects.mysql import mysqlconnector
index 0895a098d1f37c2120e4aa516d695d5f4364d6ba..5c26d8eb6d5a009ece7ceeb3c2061e51ee25fd6c 100644 (file)
@@ -90,7 +90,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest):
             # set x/y > 10
             try:
                 alt_trans.execute(update(A).values(x=15, y=19))
-            except (exc.InternalError, exc.OperationalError) as err:
+            except (
+                exc.InternalError,
+                exc.OperationalError,
+                exc.DatabaseError,
+            ) as err:
                 assert "Lock wait timeout exceeded" in str(err)
                 assert should_be_locked
             else:
@@ -103,7 +107,11 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest):
             # set x/y > 10
             try:
                 alt_trans.execute(update(B).values(x=15, y=19))
-            except (exc.InternalError, exc.OperationalError) as err:
+            except (
+                exc.InternalError,
+                exc.OperationalError,
+                exc.DatabaseError,
+            ) as err:
                 assert "Lock wait timeout exceeded" in str(err)
                 assert should_be_locked
             else:
index 309d0e9ebafa74111a71e677145d3d7a7568bfb4..ab1491fd69b0fe45187ec2286298794fbd2c5f4a 100644 (file)
@@ -560,7 +560,7 @@ class ExecuteTest(fixtures.TablesTest):
         "Older versions don't support cursor pickling, newer ones do",
     )
     @testing.fails_on(
-        "mysql+mysqlconnector",
+        "+mysqlconnector",
         "Exception doesn't come back exactly the same from pickle",
     )
     @testing.fails_on(
index 69b56423df6f92c82457dff1fd42e0a655981589..92fadf45dac9d360e4185f42daf9d9bd3d796de2 100644 (file)
@@ -1012,7 +1012,12 @@ class DefaultRequirements(SuiteRequirements):
 
     @property
     def arraysize(self):
-        return skip_if("+pymssql", "DBAPI is missing this attribute")
+        return skip_if(
+            [
+                no_support("+pymssql", "DBAPI is missing this attribute"),
+                no_support("+mysqlconnector", "DBAPI ignores this attribute"),
+            ]
+        )
 
     @property
     def emulated_lastrowid(self):
diff --git a/tox.ini b/tox.ini
index 1a5eb720dbba963576dc8b1dbdadae2c3eab4ef9..9fefea20970636ea86a1d4308ccbd33065f6b54e 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -38,6 +38,7 @@ extras=
      mysql: mysql
      mysql: pymysql
      mysql: mariadb_connector
+     mysql: mysql_connector
 
      oracle: oracle
      oracle: oracle_oracledb
@@ -142,8 +143,8 @@ setenv=
     memusage: WORKERS={env:TOX_WORKERS:-n2}
 
     mysql: MYSQL={env:TOX_MYSQL:--db mysql}
-    mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector}
-    mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector}
+    mysql: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver asyncmy --dbdriver aiomysql --dbdriver mariadbconnector --dbdriver mysqlconnector}
+    mysql-nogreenlet: EXTRA_MYSQL_DRIVERS={env:EXTRA_MYSQL_DRIVERS:--dbdriver mysqldb --dbdriver pymysql --dbdriver mariadbconnector --dbdriver mysqlconnector}
 
     mssql: MSSQL={env:TOX_MSSQL:--db mssql}