Try to get mysqlconnector somewhat working

Add CI support for MySQL connector and try to fix some of the
more obvious issues.  CI tests will run against MySQL 5.7
only for starters as there appear to be issues with
MySQL 8.0

Change-Id: Id8971143a8385a5c84f0646c21c4c21e793ce3a2

diff --git a/doc/build/changelog/unreleased_12/mysqlconnector_percents.rst b/doc/build/changelog/unreleased_12/mysqlconnector_percents.rst
new file mode 100644 (file)
index 0000000..c12dd5f
--- /dev/null
+++ b/doc/build/changelog/unreleased_12/mysqlconnector_percents.rst
@@ -0,0 +1,12 @@
+.. change::
+    :tags: bug, mysql
+
+    Fixed percent-sign doubling in mysql-connector-python dialect, which does
+    not require de-doubling of percent signs.   Additionally, the  mysql-
+    connector-python driver is inconsistent in how it passes the column names
+    in cursor.description, so a workaround decoder has been added to
+    conditionally decode these randomly-sometimes-bytes values to unicode only
+    if needed.  Also improved test support for mysql-connector-python, however
+    it should be noted that this driver still has issues with unicode that
+    continue to be unresolved as of yet.
+

diff --git a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py
index 117115888bf726a6673b91ff64f3530f7cc26673..f7b3bd670ae92cb98e99de79c8225851c29e9c74 100644 (file)
--- a/lib/sqlalchemy/dialects/mysql/mysqlconnector.py
+++ b/lib/sqlalchemy/dialects/mysql/mysqlconnector.py
@@ -14,11 +14,33 @@
     :url: http://dev.mysql.com/downloads/connector/python/
 
 
-Unicode
--------
+Current Issues
+--------------
 
-Please see :ref:`mysql_unicode` for current recommendations on unicode
-handling.
+The mysqlconnector driver has many issues that have gone unresolved
+for many years and it recommended that mysqlclient or pymysql be used
+if possible; as of June 27, 2018:
+
+* the values in cursor.description are randomly sent as either bytes
+  or text with no discernible pattern, so the dialect must test these
+  individually and attempt to decode
+
+* Under Python 2, the driver does not support SQL statements that contain
+  non-ascii characters within the SQL text, making it impossible to support
+  schema objects with non-ascii names; an ascii encoding error is raised.
+
+* additional random bytes-returned issues occur when running under MySQL 8.0
+  only
+
+* The driver does not accept the "utf8mb4" or "utf8mb3" charset parameters,
+  only "utf8", even though MySQL itself has deprecated this symbol
+
+* The driver produces deadlocks when trying to make use of SELECT..FOR UPDATE,
+  the reason is unknown.
+
+This list should be updated as these issues are resolved either in the
+upstream mysql-connector-python driver or if appropriate usage patterns
+are contributed to SQLAlchemy.
 
 """
 
@@ -28,6 +50,7 @@ from .base import (MySQLDialect, MySQLExecutionContext,
 
 from ... import util
 import re
+from ... import processors
 
 
 class MySQLExecutionContext_mysqlconnector(MySQLExecutionContext):
@@ -59,6 +82,13 @@ class MySQLCompiler_mysqlconnector(MySQLCompiler):
 
 
 class MySQLIdentifierPreparer_mysqlconnector(MySQLIdentifierPreparer):
+    @property
+    def _double_percents(self):
+        return self.dialect._mysqlconnector_double_percents
+
+    @_double_percents.setter
+    def _double_percents(self, value):
+        pass
 
     def _escape_identifier(self, value):
         value = value.replace(self.escape_quote, self.escape_to_quote)
@@ -98,6 +128,26 @@ class MySQLDialect_mysqlconnector(MySQLDialect):
         }
     )
 
+    def __init__(self, *arg, **kw):
+        super(MySQLDialect_mysqlconnector, self).__init__(*arg, **kw)
+
+        # hack description encoding since mysqlconnector randomly
+        # returns bytes or not
+        self._description_decoder = \
+            processors.to_conditional_unicode_processor_factory(
+                self.description_encoding
+            )
+
+    def _check_unicode_description(self, connection):
+        # hack description encoding since mysqlconnector randomly
+        # returns bytes or not
+        return False
+
+    @property
+    def description_encoding(self):
+        # total guess
+        return "latin-1"
+
     @util.memoized_property
     def supports_unicode_statements(self):
         return util.py3k or self._mysqlconnector_version_info > (2, 0)

diff --git a/test/dialect/mysql/test_for_update.py b/test/dialect/mysql/test_for_update.py
index af467f920a91b9344ea63f8801f19a38fa389e56..a8cbcfb87eb7fbefc7e34370b950c6c3e397f650 100644 (file)
--- a/test/dialect/mysql/test_for_update.py
+++ b/test/dialect/mysql/test_for_update.py
@@ -16,6 +16,7 @@ from sqlalchemy import testing
 class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest):
     __backend__ = True
     __only_on__ = 'mysql'
+    __requires__ = 'mysql_for_update',
 
     @classmethod
     def setup_classes(cls):

diff --git a/test/requirements.py b/test/requirements.py
index 3cc80318cbcb1a3714573877a555385cc2916568..57324a3f40431c5c05d0dbd373e9d47efdd32d06 100644 (file)
--- a/test/requirements.py
+++ b/test/requirements.py
@@ -1083,6 +1083,13 @@ class DefaultRequirements(SuiteRequirements):
         return skip_if(["oracle", "firebird"],
                        "non-standard SELECT scalar syntax")
 
+    @property
+    def mysql_for_update(self):
+        return skip_if(
+            "mysql+mysqlconnector",
+           "lock-sensitive operations crash on mysqlconnector"
+        )
+
     @property
     def mysql_fsp(self):
         return only_if('mysql >= 5.6.4')

diff --git a/tox.ini b/tox.ini
index a84a7cf162e53f9596906567bfb8d9a2097e0803..4e29c58762c33f985627ba7dee656320ecea40c6 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -21,6 +21,7 @@ deps=pytest
      postgresql: psycopg2>=2.7
      mysql: mysqlclient
      mysql: pymysql
+     mysql: mysql-connector-python
      # waiting for https://github.com/oracle/python-cx_Oracle/issues/75
      oracle: cx_oracle>=6.0.2,!=6.3
      oracle5: cx_oracle==5.2.1