From: Mike Bayer Date: Sat, 19 Aug 2017 22:39:08 +0000 (-0400) Subject: Support mariadb 10.2 X-Git-Tag: rel_1_1_15~20 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=917a971764d3aca120b22b39f18441562d723285;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Support mariadb 10.2 Fixed issue where CURRENT_TIMESTAMP would not reflect correctly in the MariaDB 10.2 series due to a syntax change, where the function is now represented as ``current_timestamp()``. Fixes: #4096 MariaDB 10.2 now supports CHECK constraints (warning: use version 10.2.9 or greater due to upstream issues noted in :ticket:`4097`). Reflection now takes these CHECK constraints into account when they are present in the ``SHOW CREATE TABLE`` output. Fixes: #4098 Change-Id: I8666d61814e8145ca12cbecad94019b44af868e3 (cherry picked from commit 29b752f8b24909c9c715e1b2c5e01119d1f46aa7) --- diff --git a/doc/build/changelog/unreleased_11/4096.rst b/doc/build/changelog/unreleased_11/4096.rst new file mode 100644 index 0000000000..1ded9d5640 --- /dev/null +++ b/doc/build/changelog/unreleased_11/4096.rst @@ -0,0 +1,18 @@ +.. change: + :tags: bug, mysql + :tickets: 4096 + :versions: 1.2.0b3 + + Fixed issue where CURRENT_TIMESTAMP would not reflect correctly + in the MariaDB 10.2 series due to a syntax change, where the function + is now represented as ``current_timestamp()``. + +.. change: + :tags: bug, mysql + :tickets: 4098 + :versions: 1.2.0b3 + + MariaDB 10.2 now supports CHECK constraints (warning: use version 10.2.9 + or greater due to upstream issues noted in :ticket:`4097`). Reflection + now takes these CHECK constraints into account when they are present in + the ``SHOW CREATE TABLE`` output. diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 5e76960dec..70d4cdf106 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -1740,6 +1740,12 @@ class MySQLDialect(default.DefaultDialect): def _is_mariadb(self): return 'MariaDB' in self.server_version_info + @property + def _is_mariadb_102(self): + return self._is_mariadb and \ + self._mariadb_normalized_version_info > (10, 2) + + @property def _mariadb_normalized_version_info(self): if len(self.server_version_info) > 5: return self.server_version_info[3:] @@ -1829,8 +1835,7 @@ class MySQLDialect(default.DefaultDialect): fkeys = [] - for spec in parsed_state.constraints: - # only FOREIGN KEYs + for spec in parsed_state.fk_constraints: ref_name = spec['table'][-1] ref_schema = len(spec['table']) > 1 and \ spec['table'][-2] or schema @@ -1861,6 +1866,24 @@ class MySQLDialect(default.DefaultDialect): fkeys.append(fkey_d) return fkeys + @reflection.cache + def get_check_constraints( + self, connection, table_name, schema=None, **kw): + + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) + + return [ + {"name": spec['name'], "sqltext": spec['sqltext']} + for spec in parsed_state.ck_constraints + ] + + @reflection.cache + def get_table_comment(self, connection, table_name, schema=None, **kw): + parsed_state = self._parsed_state_or_create( + connection, table_name, schema, **kw) + return {"text": parsed_state.table_options.get('mysql_comment', None)} + @reflection.cache def get_indexes(self, connection, table_name, schema=None, **kw): diff --git a/lib/sqlalchemy/dialects/mysql/reflection.py b/lib/sqlalchemy/dialects/mysql/reflection.py index f5f09b80b3..5c2d22cd11 100644 --- a/lib/sqlalchemy/dialects/mysql/reflection.py +++ b/lib/sqlalchemy/dialects/mysql/reflection.py @@ -20,7 +20,8 @@ class ReflectedState(object): self.table_options = {} self.table_name = None self.keys = [] - self.constraints = [] + self.fk_constraints = [] + self.ck_constraints = [] @log.class_logger @@ -56,8 +57,10 @@ class MySQLTableDefinitionParser(object): util.warn("Unknown schema content: %r" % line) elif type_ == 'key': state.keys.append(spec) - elif type_ == 'constraint': - state.constraints.append(spec) + elif type_ == 'fk_constraint': + state.fk_constraints.append(spec) + elif type_ == 'ck_constraint': + state.ck_constraints.append(spec) else: pass return state @@ -76,8 +79,8 @@ class MySQLTableDefinitionParser(object): spec['columns'] = self._parse_keyexprs(spec['columns']) return 'key', spec - # CONSTRAINT - m = self._re_constraint.match(line) + # FOREIGN KEY CONSTRAINT + m = self._re_fk_constraint.match(line) if m: spec = m.groupdict() spec['table'] = \ @@ -86,7 +89,13 @@ class MySQLTableDefinitionParser(object): for c in self._parse_keyexprs(spec['local'])] spec['foreign'] = [c[0] for c in self._parse_keyexprs(spec['foreign'])] - return 'constraint', spec + return 'fk_constraint', spec + + # CHECK constraint + m = self._re_ck_constraint.match(line) + if m: + spec = m.groupdict() + return 'ck_constraint', spec # PARTITION and SUBPARTITION m = self._re_partition.match(line) @@ -314,26 +323,26 @@ class MySQLTableDefinitionParser(object): # COLUMN_FORMAT (FIXED|DYNAMIC|DEFAULT) # STORAGE (DISK|MEMORY) self._re_column = _re_compile( - r' ' - r'%(iq)s(?P(?:%(esc_fq)s|[^%(fq)s])+)%(fq)s +' - r'(?P\w+)' - r'(?:\((?P(?:\d+|\d+,\d+|' - r'(?:\x27(?:\x27\x27|[^\x27])*\x27,?)+))\))?' - r'(?: +(?PUNSIGNED))?' - r'(?: +(?PZEROFILL))?' - r'(?: +CHARACTER SET +(?P[\w_]+))?' - r'(?: +COLLATE +(?P[\w_]+))?' - r'(?: +(?P(?:NOT )?NULL))?' - r'(?: +DEFAULT +(?P' - r'(?:NULL|\x27(?:\x27\x27|[^\x27])*\x27|\w+' - r'(?: +ON UPDATE \w+)?)' - r'))?' - r'(?: +(?PAUTO_INCREMENT))?' - r'(?: +COMMENT +(P(?:\x27\x27|[^\x27])+))?' - r'(?: +COLUMN_FORMAT +(?P\w+))?' - r'(?: +STORAGE +(?P\w+))?' - r'(?: +(?P.*))?' - r',?$' + r" " + r"%(iq)s(?P(?:%(esc_fq)s|[^%(fq)s])+)%(fq)s +" + r"(?P\w+)" + r"(?:\((?P(?:\d+|\d+,\d+|" + r"(?:'(?:''|[^'])*',?)+))\))?" + r"(?: +(?PUNSIGNED))?" + r"(?: +(?PZEROFILL))?" + r"(?: +CHARACTER SET +(?P[\w_]+))?" + r"(?: +COLLATE +(?P[\w_]+))?" + r"(?: +(?P(?:NOT )?NULL))?" + r"(?: +DEFAULT +(?P" + r"(?:NULL|'(?:''|[^'])*'|[\w\(\)]+" + r"(?: +ON UPDATE [\w\(\)]+)?)" + r"))?" + r"(?: +(?PAUTO_INCREMENT))?" + r"(?: +COMMENT +'(?P(?:''|[^'])*)')?" + r"(?: +COLUMN_FORMAT +(?P\w+))?" + r"(?: +STORAGE +(?P\w+))?" + r"(?: +(?P.*))?" + r",?$" % quotes ) @@ -372,7 +381,7 @@ class MySQLTableDefinitionParser(object): # unique constraints come back as KEYs kw = quotes.copy() kw['on'] = 'RESTRICT|CASCADE|SET NULL|NOACTION' - self._re_constraint = _re_compile( + self._re_fk_constraint = _re_compile( r' ' r'CONSTRAINT +' r'%(iq)s(?P(?:%(esc_fq)s|[^%(fq)s])+)%(fq)s +' @@ -387,6 +396,19 @@ class MySQLTableDefinitionParser(object): % kw ) + # CONSTRAINT `CONSTRAINT_1` CHECK (`x` > 5)' + # testing on MariaDB 10.2 shows that the CHECK constraint + # is returned on a line by itself, so to match without worrying + # about parenthesis in the expresion we go to the end of the line + self._re_ck_constraint = _re_compile( + r' ' + r'CONSTRAINT +' + r'%(iq)s(?P(?:%(esc_fq)s|[^%(fq)s])+)%(fq)s +' + r'CHECK +' + r'\((?P.+)\),?' + % kw + ) + # PARTITION # # punt! diff --git a/test/dialect/mysql/test_reflection.py b/test/dialect/mysql/test_reflection.py index 9e122e680f..5caec72c97 100644 --- a/test/dialect/mysql/test_reflection.py +++ b/test/dialect/mysql/test_reflection.py @@ -12,6 +12,8 @@ from sqlalchemy.dialects.mysql import base as mysql from sqlalchemy.dialects.mysql import reflection as _reflection from sqlalchemy.testing import fixtures, AssertsExecutionResults from sqlalchemy import testing +from sqlalchemy.testing import assert_raises_message, expect_warnings +import re class TypeReflectionTest(fixtures.TestBase): @@ -231,9 +233,9 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): assert reflected.c.c5.default is None assert reflected.c.c5.server_default is None assert reflected.c.c6.default is None - eq_( - str(reflected.c.c6.server_default.arg).upper(), - "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP" + assert re.match( + r"CURRENT_TIMESTAMP(\(\))? ON UPDATE CURRENT_TIMESTAMP(\(\))?", + str(reflected.c.c6.server_default.arg).upper() ) reflected.create() try: @@ -249,9 +251,9 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): assert reflected.c.c5.default is None assert reflected.c.c5.server_default is None assert reflected.c.c6.default is None - eq_( - str(reflected.c.c6.server_default.arg).upper(), - "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP" + assert re.match( + r"CURRENT_TIMESTAMP(\(\))? ON UPDATE CURRENT_TIMESTAMP(\(\))?", + str(reflected.c.c6.server_default.arg).upper() ) def test_reflection_with_table_options(self): @@ -478,6 +480,11 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): for d in inspect(testing.db).get_columns("nn_t%d" % idx) ) + if testing.db.dialect._is_mariadb_102: + current_timestamp = "current_timestamp()" + else: + current_timestamp = "CURRENT_TIMESTAMP" + eq_( reflected, [ @@ -486,15 +493,19 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): {'name': 'z', 'nullable': True, 'default': None}, {'name': 'q', 'nullable': True, 'default': None}, {'name': 'p', 'nullable': True, - 'default': 'CURRENT_TIMESTAMP'}, + 'default': current_timestamp}, {'name': 'r', 'nullable': False, - 'default': "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"}, + 'default': + "%(current_timestamp)s ON UPDATE %(current_timestamp)s" % + {"current_timestamp": current_timestamp}}, {'name': 's', 'nullable': False, - 'default': 'CURRENT_TIMESTAMP'}, + 'default': current_timestamp}, {'name': 't', 'nullable': False, - 'default': "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"}, + 'default': + "%(current_timestamp)s ON UPDATE %(current_timestamp)s" % + {"current_timestamp": current_timestamp}}, {'name': 'u', 'nullable': False, - 'default': 'CURRENT_TIMESTAMP'}, + 'default': current_timestamp}, ] ) @@ -532,6 +543,8 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): class RawReflectionTest(fixtures.TestBase): + __backend__ = True + def setup(self): dialect = mysql.dialect() self.parser = _reflection.MySQLTableDefinitionParser( @@ -568,7 +581,7 @@ class RawReflectionTest(fixtures.TestBase): " KEY (`id`) USING BTREE COMMENT 'prefix''text''suffix'") def test_fk_reflection(self): - regex = self.parser._re_constraint + regex = self.parser._re_fk_constraint m = regex.match(' CONSTRAINT `addresses_user_id_fkey` ' 'FOREIGN KEY (`user_id`) ' diff --git a/test/engine/test_reflection.py b/test/engine/test_reflection.py index e2a204e7b1..79c370b8ca 100644 --- a/test/engine/test_reflection.py +++ b/test/engine/test_reflection.py @@ -7,7 +7,7 @@ from sqlalchemy.testing import ( ComparesTables, engines, AssertsCompiledSQL, fixtures, skip) from sqlalchemy.testing.schema import Table, Column -from sqlalchemy.testing import eq_, is_true, assert_raises, \ +from sqlalchemy.testing import eq_, eq_regex, is_true, assert_raises, \ assert_raises_message from sqlalchemy import testing from sqlalchemy.util import ue @@ -1027,7 +1027,7 @@ class ReflectionTest(fixtures.TestBase, ComparesTables): const for const in t2.constraints if isinstance(const, sa.CheckConstraint)][0] - eq_(ck.sqltext.text, "q > 10") + eq_regex(ck.sqltext.text, r".?q.? > 10") eq_(ck.name, "ck1") @testing.provide_metadata diff --git a/test/requirements.py b/test/requirements.py index 58ba0bc184..52fe37f9f9 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -51,14 +51,8 @@ class DefaultRequirements(SuiteRequirements): def enforces_check_constraints(self): """Target database must also enforce check constraints.""" - def mysql_not_mariadb_102(config): - return against(config, "mysql") and ( - not config.db.dialect._is_mariadb or - config.db.dialect.server_version_info < (5, 5, 5, 10, 2) - ) - return self.check_constraints + fails_on( - mysql_not_mariadb_102, + self._mysql_not_mariadb_102, "check constraints don't enforce on MySQL, MariaDB<10.2" ) @@ -337,9 +331,8 @@ class DefaultRequirements(SuiteRequirements): @property def check_constraint_reflection(self): return fails_on_everything_except( - "postgresql", - "sqlite" - ) + "postgresql", "sqlite", self._mariadb_102 + ) @property def temp_table_names(self): @@ -930,6 +923,17 @@ class DefaultRequirements(SuiteRequirements): return only_if(check) + def _mariadb_102(self, config): + return against(config, "mysql") and \ + config.db.dialect._is_mariadb and \ + config.db.dialect._mariadb_normalized_version_info > (10, 2) + + def _mysql_not_mariadb_102(self, config): + return against(config, "mysql") and ( + not config.db.dialect._is_mariadb or + config.db.dialect._mariadb_normalized_version_info < (10, 2) + ) + def _has_mysql_on_windows(self, config): return against(config, 'mysql') and \ config.db.dialect._detect_casing(config.db) == 1 diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 50d6ddf923..2509b8cf88 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -1367,7 +1367,12 @@ class EnumTest(AssertsCompiledSQL, fixtures.TablesTest): @testing.requires.enforces_check_constraints def test_check_constraint(self): assert_raises( - (exc.IntegrityError, exc.ProgrammingError), + ( + exc.IntegrityError, exc.ProgrammingError, + exc.OperationalError, + # PyMySQL raising InternalError until + # https://github.com/PyMySQL/PyMySQL/issues/607 is resolved + exc.InternalError), testing.db.execute, "insert into non_native_enum_table " "(id, someenum) values(1, 'four')")