From: Mike Bayer Date: Fri, 14 Aug 2020 04:58:56 +0000 (-0400) Subject: Bump minimum MySQL version to 5.0.2; use all-numeric server version X-Git-Tag: rel_1_4_0b1~169^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fc97854f69ee589774627f14ce78bb8a1bbb3236;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Bump minimum MySQL version to 5.0.2; use all-numeric server version MySQL dialect's server_version_info tuple is now all numeric. String tokens like "MariaDB" are no longer present so that numeric comparison works in all cases. The .is_mariadb flag on the dialect should be consulted for whether or not mariadb was detected. Additionally removed structures meant to support extremely old MySQL versions 3.x and 4.x; the minimum MySQL version supported is now version 5.0.2. In addition, as the "MariaDB" name goes away from server version, expand upon the change in I330815ebe572b6a9818377da56621397335fa702 to support the name "mariadb" throughout the dialect and test suite when mariadb-only mode is used. This changes the "name" field on the MariaDB dialect to "mariadb", which then implies a change throughout the testing requirements system as well as all the dialect-specific DDL argument names such as "mysql_engine" is now specified as "mariadb_engine", etc. Make use of the recent additions to test suite URL provisioning so that we can force MariaDB databases to have a "mariadb-only" dialect which allows us to test this name change fully. Update documentation to refer to MySQL / MariaDB explicitly as well as indicating the "mariadb_" prefix used for options. It seems likely that MySQL and MariaDB version numbers are going to start colliding at some point so having the "mariadb" name be available as a totally separate dialect name should give us some options in this regard. Currently also includes a date related fix to a test for the postgresql dialect that was implicitly assuming a non-UTC timezone Fixes: #4189 Change-Id: I00e76d00f62971e1f067bd61915fa6cc1cf64e5e --- diff --git a/doc/build/changelog/unreleased_14/4189.rst b/doc/build/changelog/unreleased_14/4189.rst new file mode 100644 index 0000000000..4eb0df6295 --- /dev/null +++ b/doc/build/changelog/unreleased_14/4189.rst @@ -0,0 +1,11 @@ +.. change:: + :tags: bug, mysql + :tickets: 4189 + + MySQL dialect's server_version_info tuple is now all numeric. String + tokens like "MariaDB" are no longer present so that numeric comparison + works in all cases. The .is_mariadb flag on the dialect should be + consulted for whether or not mariadb was detected. Additionally removed + structures meant to support extremely old MySQL versions 3.x and 4.x; + the minimum MySQL version supported is now version 5.0.2. + diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index de75f4104b..34afc81a7e 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -13,13 +13,12 @@ r""" Supported Versions and Features ------------------------------- -SQLAlchemy supports MySQL starting with version 4.1 through modern releases, as -well as all modern versions of MariaDB. However, no heroic measures are taken -to work around major missing SQL features - if your server version does not -support sub-selects, for example, they won't work in SQLAlchemy either. +SQLAlchemy supports MySQL starting with version 5.0.2 through modern releases, +as well as all modern versions of MariaDB. See the official MySQL +documentation for detailed information about features supported in any given +server release. -See the official MySQL documentation for detailed information about features -supported in any given server release. +.. versionchanged:: 1.4 minimum MySQL version supported is now 5.0.2. MariaDB Support ~~~~~~~~~~~~~~~ @@ -39,7 +38,12 @@ backing database reports as MariaDB. Based on this flag, the dialect can make different choices in those of areas where its behavior must be different. -The dialect also supports a "MariaDB-only" mode of connection, which may be +.. _mysql_mariadb_only_mode: + +MariaDB-Only Mode +~~~~~~~~~~~~~~~~~ + +The dialect also supports an **optional** "MariaDB-only" mode of connection, which may be useful for the case where an application makes use of MariaDB-specific features and is not compatible with a MySQL database. To use this mode of operation, replace the "mysql" token in the above URL with "mariadb":: @@ -49,6 +53,32 @@ replace the "mysql" token in the above URL with "mariadb":: The above engine, upon first connect, will raise an error if the server version detection detects that the backing database is not MariaDB. +When using an engine with ``"mariadb"`` as the dialect name, **all mysql-specific options +that include the name "mysql" in them are now named with "mariadb"**. This means +options like ``mysql_engine`` should be named ``mariadb_engine``, etc. Both +"mysql" and "mariadb" options can be used simultaneously for applications that +use URLs with both "mysql" and "mariadb" dialects:: + + my_table = Table( + "mytable", + metadata, + Column("id", Integer, primary_key=True), + Column("textdata", String(50)), + mariadb_engine="InnoDB", + mysql_engine="InnoDB", + ) + + Index( + "textdata_ix", + my_table.c.textdata, + mysql_prefix="FULLTEXT", + mariadb_prefix="FULLTEXT", + ) + +Similar behavior will occur when the above structures are reflected, i.e. the +"mariadb" prefix will be present in the option names when the database URL +is based on the "mariadb" name. + .. versionadded:: 1.4 Added "mariadb" dialect name supporting "MariaDB-only mode" for the MySQL dialect. @@ -57,7 +87,7 @@ detection detects that the backing database is not MariaDB. Connection Timeouts and Disconnects ----------------------------------- -MySQL features an automatic connection close behavior, for connections that +MySQL / MariaDB feature an automatic connection close behavior, for connections that have been idle for a fixed period of time, defaulting to eight hours. To circumvent having this issue, use the :paramref:`_sa.create_engine.pool_recycle` option which ensures that @@ -80,7 +110,7 @@ be employed. See :ref:`pool_disconnects` for current approaches. CREATE TABLE arguments including Storage Engines ------------------------------------------------ -MySQL's CREATE TABLE syntax includes a wide array of special options, +Both MySQL's and MariaDB's CREATE TABLE syntax includes a wide array of special options, including ``ENGINE``, ``CHARSET``, ``MAX_ROWS``, ``ROW_FORMAT``, ``INSERT_METHOD``, and many more. To accommodate the rendering of these arguments, specify the form @@ -95,7 +125,28 @@ of ``1024``:: mysql_key_block_size="1024" ) -The MySQL dialect will normally transfer any keyword specified as +When supporing :ref:`mysql_mariadb_only_mode` mode, similar keys against +the "mariadb" prefix must be included as well. The values can of course +vary independently so that different settings on MySQL vs. MariaDB may +be maintained:: + + # support both "mysql" and "mariadb-only" engine URLs + + Table('mytable', metadata, + Column('data', String(32)), + + mysql_engine='InnoDB', + mariadb_engine='InnoDB', + + mysql_charset='utf8mb4', + mariadb_charset='utf8', + + mysql_key_block_size="1024" + mariadb_key_block_size="1024" + + ) + +The MySQL / MariaDB dialects will normally transfer any keyword specified as ``mysql_keyword_name`` to be rendered as ``KEYWORD_NAME`` in the ``CREATE TABLE`` statement. A handful of these names will render with a space instead of an underscore; to support this, the MySQL dialect has awareness of @@ -111,7 +162,7 @@ to ``InnoDB``. The ``InnoDB`` engine is typically preferred for its support of transactions and foreign keys. A :class:`_schema.Table` -that is created in a MySQL database with a storage engine +that is created in a MySQL / MariaDB database with a storage engine of ``MyISAM`` will be essentially non-transactional, meaning any INSERT/UPDATE/DELETE statement referring to this table will be invoked as autocommit. It also will have no support for foreign key constraints; while @@ -123,16 +174,11 @@ For fully atomic transactions as well as support for foreign key constraints, all participating ``CREATE TABLE`` statements must specify a transactional engine, which in the vast majority of cases is ``InnoDB``. -.. seealso:: - - `The InnoDB Storage Engine - `_ - - on the MySQL website. Case Sensitivity and Table Reflection ------------------------------------- -MySQL has inconsistent support for case-sensitive identifier +Both MySQL and MariaDB have inconsistent support for case-sensitive identifier names, basing support on specific details of the underlying operating system. However, it has been observed that no matter what case sensitivity behavior is present, the names of tables in @@ -141,7 +187,7 @@ as all-lower case, making it impossible to accurately reflect a schema where inter-related tables use mixed-case identifier names. Therefore it is strongly advised that table names be declared as -all lower case both within SQLAlchemy as well as on the MySQL +all lower case both within SQLAlchemy as well as on the MySQL / MariaDB database itself, especially if database reflection features are to be used. @@ -150,7 +196,7 @@ to be used. Transaction Isolation Level --------------------------- -All MySQL dialects support setting of transaction isolation level both via a +All MySQL / MariaDB dialects support setting of transaction isolation level both via a dialect-specific parameter :paramref:`_sa.create_engine.isolation_level` accepted by :func:`_sa.create_engine`, as well as the @@ -186,7 +232,7 @@ Valid values for ``isolation_level`` include: The special ``AUTOCOMMIT`` value makes use of the various "autocommit" attributes provided by specific DBAPIs, and is currently supported by MySQLdb, MySQL-Client, MySQL-Connector Python, and PyMySQL. Using it, -the MySQL connection will return true for the value of +the database connection will return true for the value of ``SELECT @@autocommit;``. .. seealso:: @@ -226,7 +272,7 @@ Server Side Cursors ------------------- Server-side cursor support is available for the MySQLdb and PyMySQL dialects. -From a MySQL point of view this means that the ``MySQLdb.cursors.SSCursor`` or +From a database driver point of view this means that the ``MySQLdb.cursors.SSCursor`` or ``pymysql.cursors.SSCursor`` class is used when building up the cursor which will receive results. The most typical way of invoking this feature is via the :paramref:`.Connection.execution_options.stream_results` connection execution @@ -244,7 +290,7 @@ Unicode Charset Selection ~~~~~~~~~~~~~~~~~ -Most MySQL DBAPIs offer the option to set the client character set for +Most MySQL / MariaDB DBAPIs offer the option to set the client character set for a connection. This is typically delivered using the ``charset`` parameter in the URL, such as:: @@ -257,17 +303,16 @@ will make use of the ``default-character-set`` setting in the ``my.cnf`` file as well. Documentation for the DBAPI in use should be consulted for specific behavior. -The encoding used for Unicode has traditionally been ``'utf8'``. However, -for MySQL versions 5.5.3 on forward, a new MySQL-specific encoding -``'utf8mb4'`` has been introduced, and as of MySQL 8.0 a warning is emitted -by the server if plain ``utf8`` is specified within any server-side -directives, replaced with ``utf8mb3``. The rationale for this new encoding -is due to the fact that MySQL's legacy utf-8 encoding only supports -codepoints up to three bytes instead of four. Therefore, -when communicating with a MySQL database -that includes codepoints more than three bytes in size, -this new charset is preferred, if supported by both the database as well -as the client DBAPI, as in:: +The encoding used for Unicode has traditionally been ``'utf8'``. However, for +MySQL versions 5.5.3 and MariaDB 5.5 on forward, a new MySQL-specific encoding +``'utf8mb4'`` has been introduced, and as of MySQL 8.0 a warning is emitted by +the server if plain ``utf8`` is specified within any server-side directives, +replaced with ``utf8mb3``. The rationale for this new encoding is due to the +fact that MySQL's legacy utf-8 encoding only supports codepoints up to three +bytes instead of four. Therefore, when communicating with a MySQL or MariaDB +database that includes codepoints more than three bytes in size, this new +charset is preferred, if supported by both the database as well as the client +DBAPI, as in:: e = create_engine( "mysql+pymysql://scott:tiger@localhost/test?charset=utf8mb4") @@ -275,7 +320,7 @@ as the client DBAPI, as in:: All modern DBAPIs should support the ``utf8mb4`` charset. In order to use ``utf8mb4`` encoding for a schema that was created with legacy -``utf8``, changes to the MySQL schema and/or server configuration may be +``utf8``, changes to the MySQL/MariaDB schema and/or server configuration may be required. .. seealso:: @@ -334,7 +379,7 @@ most efficient place for this additional keyword to be passed. ANSI Quoting Style ------------------ -MySQL features two varieties of identifier "quoting style", one using +MySQL / MariaDB feature two varieties of identifier "quoting style", one using backticks and the other using quotes, e.g. ```some_identifier``` vs. ``"some_identifier"``. All MySQL dialects detect which version is in use by checking the value of ``sql_mode`` when a connection is first @@ -344,18 +389,18 @@ into play when rendering table and column names as well as when reflecting existing database structures. The detection is entirely automatic and no special configuration is needed to use either quoting style. -MySQL SQL Extensions --------------------- +MySQL / MariaDB SQL Extensions +------------------------------ -Many of the MySQL SQL extensions are handled through SQLAlchemy's generic +Many of the MySQL / MariaDB SQL extensions are handled through SQLAlchemy's generic function and operator support:: table.select(table.c.password==func.md5('plaintext')) table.select(table.c.username.op('regexp')('^[a-d]')) -And of course any valid MySQL statement can be executed as a string as well. +And of course any valid SQL statement can be executed as a string as well. -Some limited direct support for MySQL extensions to SQL is currently +Some limited direct support for MySQL / MariaDB extensions to SQL is currently available. * INSERT..ON DUPLICATE KEY UPDATE: See @@ -368,7 +413,7 @@ available. * UPDATE with LIMIT:: - update(..., mysql_limit=10) + update(..., mysql_limit=10, mariadb_limit=10) * optimizer hints, use :meth:`_expression.Select.prefix_with` and :meth:`_query.Query.prefix_with`:: @@ -385,7 +430,7 @@ available. INSERT...ON DUPLICATE KEY UPDATE (Upsert) ------------------------------------------ -MySQL allows "upserts" (update or insert) +MySQL / MariaDB allow "upserts" (update or insert) of rows into a table via the ``ON DUPLICATE KEY UPDATE`` clause of the ``INSERT`` statement. A candidate row will only be inserted if that row does not match an existing primary or unique key in the table; otherwise, an UPDATE @@ -505,63 +550,36 @@ This setting is currently hardcoded. :attr:`_engine.CursorResult.rowcount` -CAST Support ------------- - -MySQL documents the CAST operator as available in version 4.0.2. When using -the SQLAlchemy :func:`.cast` function, SQLAlchemy -will not render the CAST token on MySQL before this version, based on server -version detection, instead rendering the internal expression directly. - -CAST may still not be desirable on an early MySQL version post-4.0.2, as it -didn't add all datatype support until 4.1.1. If your application falls into -this narrow area, the behavior of CAST can be controlled using the -:ref:`sqlalchemy.ext.compiler_toplevel` system, as per the recipe below:: - - from sqlalchemy.sql.expression import Cast - from sqlalchemy.ext.compiler import compiles - - @compiles(Cast, 'mysql') - def _check_mysql_version(element, compiler, **kw): - if compiler.dialect.server_version_info < (4, 1, 0): - return compiler.process(element.clause, **kw) - else: - return compiler.visit_cast(element, **kw) - -The above function, which only needs to be declared once -within an application, overrides the compilation of the -:func:`.cast` construct to check for version 4.1.0 before -fully rendering CAST; else the internal element of the -construct is rendered directly. - - .. _mysql_indexes: -MySQL Specific Index Options ----------------------------- +MySQL / MariaDB- Specific Index Options +----------------------------------------- -MySQL-specific extensions to the :class:`.Index` construct are available. +MySQL and MariaDB-specific extensions to the :class:`.Index` construct are available. Index Length ~~~~~~~~~~~~~ -MySQL provides an option to create index entries with a certain length, where +MySQL and MariaDB both provide an option to create index entries with a certain length, where "length" refers to the number of characters or bytes in each value which will become part of the index. SQLAlchemy provides this feature via the -``mysql_length`` parameter:: +``mysql_length`` and/or ``mariadb_length`` parameters:: - Index('my_index', my_table.c.data, mysql_length=10) + Index('my_index', my_table.c.data, mysql_length=10, mariadb_length=10) Index('a_b_idx', my_table.c.a, my_table.c.b, mysql_length={'a': 4, 'b': 9}) + Index('a_b_idx', my_table.c.a, my_table.c.b, mariadb_length={'a': 4, + 'b': 9}) + Prefix lengths are given in characters for nonbinary string types and in bytes for binary string types. The value passed to the keyword argument *must* be either an integer (and, thus, specify the same prefix length value for all columns of the index) or a dict in which keys are column names and values are -prefix length values for corresponding columns. MySQL only allows a length for -a column of an index if it is for a CHAR, VARCHAR, TEXT, BINARY, VARBINARY and -BLOB. +prefix length values for corresponding columns. MySQL and MariaDB only allow a +length for a column of an index if it is for a CHAR, VARCHAR, TEXT, BINARY, +VARBINARY and BLOB. Index Prefixes ~~~~~~~~~~~~~~ @@ -589,11 +607,11 @@ Some MySQL storage engines permit you to specify an index type when creating an index or primary key constraint. SQLAlchemy provides this feature via the ``mysql_using`` parameter on :class:`.Index`:: - Index('my_index', my_table.c.data, mysql_using='hash') + Index('my_index', my_table.c.data, mysql_using='hash', mariadb_using='hash') As well as the ``mysql_using`` parameter on :class:`.PrimaryKeyConstraint`:: - PrimaryKeyConstraint("data", mysql_using='hash') + PrimaryKeyConstraint("data", mysql_using='hash', mariadb_using='hash') The value passed to the keyword argument will be simply passed through to the underlying CREATE INDEX or PRIMARY KEY clause, so it *must* be a valid index @@ -613,57 +631,51 @@ is available using the keyword argument ``mysql_with_parser``:: Index( 'my_index', my_table.c.data, - mysql_prefix='FULLTEXT', mysql_with_parser="ngram") + mysql_prefix='FULLTEXT', mysql_with_parser="ngram", + mariadb_prefix='FULLTEXT', mariadb_with_parser="ngram", + ) .. versionadded:: 1.3 .. _mysql_foreign_keys: -MySQL Foreign Keys ------------------- +MySQL / MariaDB Foreign Keys +----------------------------- -MySQL's behavior regarding foreign keys has some important caveats. +MySQL and MariaDB's behavior regarding foreign keys has some important caveats. Foreign Key Arguments to Avoid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -MySQL does not support the foreign key arguments "DEFERRABLE", "INITIALLY", +Neither MySQL nor MariaDB support the foreign key arguments "DEFERRABLE", "INITIALLY", or "MATCH". Using the ``deferrable`` or ``initially`` keyword argument with :class:`_schema.ForeignKeyConstraint` or :class:`_schema.ForeignKey` will have the effect of these keywords being rendered in a DDL expression, which will then raise an -error on MySQL. In order to use these keywords on a foreign key while having -them ignored on a MySQL backend, use a custom compile rule:: +error on MySQL or MariaDB. In order to use these keywords on a foreign key while having +them ignored on a MySQL / MariaDB backend, use a custom compile rule:: from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import ForeignKeyConstraint - @compiles(ForeignKeyConstraint, "mysql") + @compiles(ForeignKeyConstraint, "mysql", "mariadb") def process(element, compiler, **kw): element.deferrable = element.initially = None return compiler.visit_foreign_key_constraint(element, **kw) -.. versionchanged:: 0.9.0 - the MySQL backend no longer silently ignores - the ``deferrable`` or ``initially`` keyword arguments of - :class:`_schema.ForeignKeyConstraint` and :class:`_schema.ForeignKey`. - The "MATCH" keyword is in fact more insidious, and is explicitly disallowed -by SQLAlchemy in conjunction with the MySQL backend. This argument is -silently ignored by MySQL, but in addition has the effect of ON UPDATE and ON +by SQLAlchemy in conjunction with the MySQL or MariaDB backends. This argument is +silently ignored by MySQL / MariaDB, but in addition has the effect of ON UPDATE and ON DELETE options also being ignored by the backend. Therefore MATCH should -never be used with the MySQL backend; as is the case with DEFERRABLE and -INITIALLY, custom compilation rules can be used to correct a MySQL +never be used with the MySQL / MariaDB backends; as is the case with DEFERRABLE and +INITIALLY, custom compilation rules can be used to correct a ForeignKeyConstraint at DDL definition time. -.. versionadded:: 0.9.0 - the MySQL backend will raise a - :class:`.CompileError` when the ``match`` keyword is used with - :class:`_schema.ForeignKeyConstraint` or :class:`_schema.ForeignKey`. - Reflection of Foreign Key Constraints ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Not all MySQL storage engines support foreign keys. When using the +Not all MySQL / MariaDB storage engines support foreign keys. When using the very common ``MyISAM`` MySQL storage engine, the information loaded by table reflection will not include foreign keys. For these tables, you may supply a :class:`~sqlalchemy.ForeignKeyConstraint` at reflection time:: @@ -679,22 +691,22 @@ reflection will not include foreign keys. For these tables, you may supply a .. _mysql_unique_constraints: -MySQL Unique Constraints and Reflection ---------------------------------------- +MySQL / MariaDB Unique Constraints and Reflection +---------------------------------------------------- SQLAlchemy supports both the :class:`.Index` construct with the flag ``unique=True``, indicating a UNIQUE index, as well as the :class:`.UniqueConstraint` construct, representing a UNIQUE constraint. -Both objects/syntaxes are supported by MySQL when emitting DDL to create -these constraints. However, MySQL does not have a unique constraint +Both objects/syntaxes are supported by MySQL / MariaDB when emitting DDL to create +these constraints. However, MySQL / MariaDB does not have a unique constraint construct that is separate from a unique index; that is, the "UNIQUE" -constraint on MySQL is equivalent to creating a "UNIQUE INDEX". +constraint on MySQL / MariaDB is equivalent to creating a "UNIQUE INDEX". When reflecting these constructs, the :meth:`_reflection.Inspector.get_indexes` and the :meth:`_reflection.Inspector.get_unique_constraints` methods will **both** -return an entry for a UNIQUE index in MySQL. However, when performing +return an entry for a UNIQUE index in MySQL / MariaDB. However, when performing full table reflection using ``Table(..., autoload=True)``, the :class:`.UniqueConstraint` construct is **not** part of the fully reflected :class:`_schema.Table` construct under any @@ -708,10 +720,10 @@ TIMESTAMP / DATETIME issues .. _mysql_timestamp_onupdate: -Rendering ON UPDATE CURRENT TIMESTAMP for MySQL's explicit_defaults_for_timestamp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Rendering ON UPDATE CURRENT TIMESTAMP for MySQL / MariaDB's explicit_defaults_for_timestamp +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -MySQL has historically expanded the DDL for the :class:`_types.TIMESTAMP` +MySQL / MariaDB have historically expanded the DDL for the :class:`_types.TIMESTAMP` datatype into the phrase "TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP", which includes non-standard SQL that automatically updates the column with the current timestamp when an UPDATE occurs, eliminating the @@ -873,7 +885,6 @@ output:: from array import array as _array from collections import defaultdict import re -import sys from sqlalchemy import literal_column from sqlalchemy.sql import visitors @@ -1545,14 +1556,6 @@ class MySQLCompiler(compiler.SQLCompiler): return None def visit_cast(self, cast, **kw): - # No cast until 4, no decimals until 5. - if not self.dialect._supports_cast: - util.warn( - "Current MySQL version does not support " - "CAST; the CAST will be skipped." - ) - return self.process(cast.clause.self_group(), **kw) - type_ = self.process(cast.typeclause) if type_ is None: util.warn( @@ -1902,13 +1905,13 @@ class MySQLDDLCompiler(compiler.DDLCompiler): if index.unique: text += "UNIQUE " - index_prefix = index.kwargs.get("mysql_prefix", None) + index_prefix = index.kwargs.get("%s_prefix" % self.dialect.name, None) if index_prefix: text += index_prefix + " " text += "INDEX %s ON %s " % (name, table) - length = index.dialect_options["mysql"]["length"] + length = index.dialect_options[self.dialect.name]["length"] if length is not None: if isinstance(length, dict): @@ -2384,6 +2387,7 @@ class MySQLDialect(default.DefaultDialect): preparer = MySQLIdentifierPreparer is_mariadb = False + _mariadb_normalized_version_info = None # default SQL compilation settings - # these are modified upon initialize(), @@ -2482,6 +2486,24 @@ class MySQLDialect(default.DefaultDialect): val = val.decode() return val.upper().replace("-", " ") + @classmethod + def _is_mariadb_from_url(cls, url): + dbapi = cls.dbapi() + dialect = cls(dbapi=dbapi) + + cargs, cparams = dialect.create_connect_args(url) + conn = dialect.connect(*cargs, **cparams) + try: + cursor = conn.cursor() + cursor.execute("SELECT VERSION() LIKE '%MariaDB%'") + val = cursor.fetchone()[0] + except: + raise + else: + return bool(val) + finally: + conn.close() + def _get_server_version_info(self, connection): # get database server version info explicitly over the wire # to avoid proxy servers like MaxScale getting in the @@ -2498,23 +2520,38 @@ class MySQLDialect(default.DefaultDialect): def _parse_server_version(self, val): version = [] - r = re.compile(r"[.\-]") - for n in r.split(val): - try: - version.append(int(n)) - except ValueError: - mariadb = re.match(r"(.*)(MariaDB)(.*)", n) - if mariadb: - version.extend(g for g in mariadb.groups() if g) - else: - version.append(n) + is_mariadb = False + + r = re.compile(r"[.\-+]") + tokens = r.split(val) + for token in tokens: + parsed_token = re.match( + r"^(?:(\d+)(?:a|b|c)?|(MariaDB\w*))$", token + ) + if not parsed_token: + continue + elif parsed_token.group(2): + self._mariadb_normalized_version_info = tuple(version[-3:]) + is_mariadb = True + else: + digit = int(parsed_token.group(1)) + version.append(digit) server_version_info = tuple(version) - self._set_mariadb( - server_version_info and "MariaDB" in server_version_info, val - ) + self._set_mariadb(server_version_info and is_mariadb, val) + + if not is_mariadb: + self._mariadb_normalized_version_info = server_version_info + + if server_version_info < (5, 0, 2): + raise NotImplementedError( + "the MySQL/MariaDB dialect supports server " + "version info 5.0.2 and above." + ) + # setting it here to help w the test suite + self.server_version_info = server_version_info return server_version_info def _set_mariadb(self, is_mariadb, server_version_info): @@ -2528,36 +2565,6 @@ class MySQLDialect(default.DefaultDialect): ) self.is_mariadb = is_mariadb - def do_commit(self, dbapi_connection): - """Execute a COMMIT.""" - - # COMMIT/ROLLBACK were introduced in 3.23.15. - # Yes, we have at least one user who has to talk to these old - # versions! - # - # Ignore commit/rollback if support isn't present, otherwise even - # basic operations via autocommit fail. - try: - dbapi_connection.commit() - except Exception: - if self.server_version_info < (3, 23, 15): - args = sys.exc_info()[1].args - if args and args[0] == 1064: - return - raise - - def do_rollback(self, dbapi_connection): - """Execute a ROLLBACK.""" - - try: - dbapi_connection.rollback() - except Exception: - if self.server_version_info < (3, 23, 15): - args = sys.exc_info()[1].args - if args and args[0] == 1064: - return - raise - def do_begin_twophase(self, connection, xid): connection.execute(sql.text("XA BEGIN :xid"), dict(xid=xid)) @@ -2772,24 +2779,6 @@ class MySQLDialect(default.DefaultDialect): 2, ) - @property - def _mariadb_normalized_version_info(self): - # MariaDB's wire-protocol prepends the server_version with - # the string "5.5"; now that we use @@version we no longer see this. - - if self.is_mariadb: - idx = self.server_version_info.index("MariaDB") - return self.server_version_info[idx - 3 : idx] - else: - return self.server_version_info - - @property - def _supports_cast(self): - return ( - self.server_version_info is None - or self.server_version_info >= (4, 0, 2) - ) - @reflection.cache def get_schema_names(self, connection, **kw): rp = connection.exec_driver_sql("SHOW schemas") @@ -2804,34 +2793,22 @@ class MySQLDialect(default.DefaultDialect): current_schema = self.default_schema_name charset = self._connection_charset - if self.server_version_info < (5, 0, 2): - rp = connection.exec_driver_sql( - "SHOW TABLES FROM %s" - % self.identifier_preparer.quote_identifier(current_schema) - ) - return [ - row[0] for row in self._compat_fetchall(rp, charset=charset) - ] - else: - rp = connection.exec_driver_sql( - "SHOW FULL TABLES FROM %s" - % self.identifier_preparer.quote_identifier(current_schema) - ) - return [ - row[0] - for row in self._compat_fetchall(rp, charset=charset) - if row[1] == "BASE TABLE" - ] + rp = connection.exec_driver_sql( + "SHOW FULL TABLES FROM %s" + % self.identifier_preparer.quote_identifier(current_schema) + ) + + return [ + row[0] + for row in self._compat_fetchall(rp, charset=charset) + if row[1] == "BASE TABLE" + ] @reflection.cache def get_view_names(self, connection, schema=None, **kw): - if self.server_version_info < (5, 0, 2): - raise NotImplementedError if schema is None: schema = self.default_schema_name - if self.server_version_info < (5, 0, 2): - return self.get_table_names(connection, schema) charset = self._connection_charset rp = connection.exec_driver_sql( "SHOW FULL TABLES FROM %s" @@ -3012,7 +2989,11 @@ class MySQLDialect(default.DefaultDialect): parsed_state = self._parsed_state_or_create( connection, table_name, schema, **kw ) - return {"text": parsed_state.table_options.get("mysql_comment", None)} + return { + "text": parsed_state.table_options.get( + "%s_comment" % self.name, None + ) + } @reflection.cache def get_indexes(self, connection, table_name, schema=None, **kw): @@ -3032,7 +3013,7 @@ class MySQLDialect(default.DefaultDialect): if flavor == "UNIQUE": unique = True elif flavor in ("FULLTEXT", "SPATIAL"): - dialect_options["mysql_prefix"] = flavor + dialect_options["%s_prefix" % self.name] = flavor elif flavor is None: pass else: @@ -3042,7 +3023,9 @@ class MySQLDialect(default.DefaultDialect): pass if spec["parser"]: - dialect_options["mysql_with_parser"] = spec["parser"] + dialect_options["%s_with_parser" % (self.name)] = spec[ + "parser" + ] index_d = {} if dialect_options: @@ -3104,11 +3087,7 @@ class MySQLDialect(default.DefaultDialect): retrieved server version information first. """ - if self.server_version_info < (4, 1) and self._server_ansiquotes: - # ANSI_QUOTES doesn't affect SHOW CREATE TABLE on < 4.1 - preparer = self.preparer(self, server_ansiquotes=False) - else: - preparer = self.identifier_preparer + preparer = self.identifier_preparer return _reflection.MySQLTableDefinitionParser(self, preparer) @reflection.cache @@ -3171,13 +3150,10 @@ class MySQLDialect(default.DefaultDialect): """ collations = {} - if self.server_version_info < (4, 1, 0): - pass - else: - charset = self._connection_charset - rs = connection.exec_driver_sql("SHOW COLLATION") - for row in self._compat_fetchall(rs, charset): - collations[row[0]] = row[1] + charset = self._connection_charset + rs = connection.exec_driver_sql("SHOW COLLATION") + for row in self._compat_fetchall(rs, charset): + collations[row[0]] = row[1] return collations def _detect_sql_mode(self, connection): diff --git a/lib/sqlalchemy/dialects/mysql/mariadb.py b/lib/sqlalchemy/dialects/mysql/mariadb.py index 73db9eb225..c6cadcd603 100644 --- a/lib/sqlalchemy/dialects/mysql/mariadb.py +++ b/lib/sqlalchemy/dialects/mysql/mariadb.py @@ -3,6 +3,7 @@ from .base import MySQLDialect class MariaDBDialect(MySQLDialect): is_mariadb = True + name = "mariadb" def loader(driver): diff --git a/lib/sqlalchemy/dialects/mysql/provision.py b/lib/sqlalchemy/dialects/mysql/provision.py index bf126464d4..b86056da6c 100644 --- a/lib/sqlalchemy/dialects/mysql/provision.py +++ b/lib/sqlalchemy/dialects/mysql/provision.py @@ -1,10 +1,33 @@ +import copy + +from ... import exc from ...testing.provision import configure_follower from ...testing.provision import create_db from ...testing.provision import drop_db +from ...testing.provision import generate_driver_url from ...testing.provision import temp_table_keyword_args -@create_db.for_db("mysql") +@generate_driver_url.for_db("mysql", "mariadb") +def generate_driver_url(url, driver): + backend = url.get_backend_name() + + if backend == "mysql": + dialect_cls = url.get_dialect() + if dialect_cls._is_mariadb_from_url(url): + backend = "mariadb" + + new_url = copy.copy(url) + new_url.drivername = "%s+%s" % (backend, driver) + try: + new_url.get_dialect() + except exc.NoSuchModuleError: + return None + else: + return new_url + + +@create_db.for_db("mysql", "mariadb") def _mysql_create_db(cfg, eng, ident): with eng.connect() as conn: try: @@ -23,13 +46,13 @@ def _mysql_create_db(cfg, eng, ident): ) -@configure_follower.for_db("mysql") +@configure_follower.for_db("mysql", "mariadb") def _mysql_configure_follower(config, ident): config.test_schema = "%s_test_schema" % ident config.test_schema_2 = "%s_test_schema_2" % ident -@drop_db.for_db("mysql") +@drop_db.for_db("mysql", "mariadb") def _mysql_drop_db(cfg, eng, ident): with eng.connect() as conn: conn.exec_driver_sql("DROP DATABASE %s_test_schema" % ident) @@ -37,6 +60,6 @@ def _mysql_drop_db(cfg, eng, ident): conn.exec_driver_sql("DROP DATABASE %s" % ident) -@temp_table_keyword_args.for_db("mysql") +@temp_table_keyword_args.for_db("mysql", "mariadb") def _mysql_temp_table_keyword_args(cfg, eng): return {"prefixes": ["TEMPORARY"]} diff --git a/lib/sqlalchemy/testing/provision.py b/lib/sqlalchemy/testing/provision.py index 13a5ea078c..21bacfca2f 100644 --- a/lib/sqlalchemy/testing/provision.py +++ b/lib/sqlalchemy/testing/provision.py @@ -21,9 +21,10 @@ class register(object): def init(cls, fn): return register().for_db("*")(fn) - def for_db(self, dbname): + def for_db(self, *dbnames): def decorate(fn): - self.fns[dbname] = fn + for dbname in dbnames: + self.fns[dbname] = fn return self return decorate @@ -138,6 +139,7 @@ def _generate_driver_urls(url, extra_drivers): main_driver = url.get_driver_name() extra_drivers.discard(main_driver) + url = generate_driver_url(url, main_driver) yield str(url) for drv in list(extra_drivers): diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index 36d0ce4c61..3d3980b305 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -19,7 +19,6 @@ import platform import sys from . import exclusions -from . import fails_on_everything_except from .. import util @@ -407,6 +406,15 @@ class SuiteRequirements(Requirements): """ return exclusions.closed() + @property + def emulated_lastrowid_even_with_sequences(self): + """"target dialect retrieves cursor.lastrowid or an equivalent + after an insert() construct executes, even if the table has a + Sequence on it. + + """ + return exclusions.closed() + @property def dbapi_lastrowid(self): """"target platform includes a 'lastrowid' accessor on the DBAPI @@ -1246,13 +1254,3 @@ class SuiteRequirements(Requirements): lambda config: not config.db.dialect.supports_is_distinct_from, "driver doesn't support an IS DISTINCT FROM construct", ) - - @property - def emulated_lastrowid_even_with_sequences(self): - """"target dialect retrieves cursor.lastrowid or an equivalent - after an insert() construct executes, even if the table has a - Sequence on it.. - """ - return fails_on_everything_except( - "mysql", "sqlite+pysqlite", "sqlite+pysqlcipher", "sybase", - ) diff --git a/lib/sqlalchemy/testing/schema.py b/lib/sqlalchemy/testing/schema.py index ab527cae3b..f5bd1f7a23 100644 --- a/lib/sqlalchemy/testing/schema.py +++ b/lib/sqlalchemy/testing/schema.py @@ -33,6 +33,16 @@ def Table(*args, **kw): kw["mysql_engine"] = "InnoDB" else: kw["mysql_engine"] = "MyISAM" + elif exclusions.against(config._current, "mariadb"): + if ( + "mariadb_engine" not in kw + and "mariadb_type" not in kw + and "autoload_with" not in kw + ): + if "test_needs_fk" in test_opts or "test_needs_acid" in test_opts: + kw["mariadb_engine"] = "InnoDB" + else: + kw["mariadb_engine"] = "MyISAM" # Apply some default cascading rules for self-referential foreign keys. # MySQL InnoDB has some issues around selecting self-refs too. diff --git a/test/dialect/mysql/test_compiler.py b/test/dialect/mysql/test_compiler.py index 2053318b6e..aca1db33ce 100644 --- a/test/dialect/mysql/test_compiler.py +++ b/test/dialect/mysql/test_compiler.py @@ -640,18 +640,6 @@ class SQLTest(fixtures.TestBase, AssertsCompiledSQL): with expect_warnings("Datatype FLOAT does not support CAST on MySQL;"): self.assert_compile(expr, "(foo + 5)", literal_binds=True) - dialect = mysql.MySQLDialect() - dialect.server_version_info = (3, 9, 8) - with expect_warnings("Current MySQL version does not support CAST"): - eq_( - str( - expr.compile( - dialect=dialect, compile_kwargs={"literal_binds": True} - ) - ), - "(foo + 5)", - ) - m = mysql @testing.combinations( @@ -676,33 +664,12 @@ class SQLTest(fixtures.TestBase, AssertsCompiledSQL): with expect_warnings("Datatype .* does not support CAST on MySQL;"): self.assert_compile(cast(t.c.col, type_), expected) - def test_no_cast_pre_4(self): - self.assert_compile( - cast(Column("foo", Integer), String), "CAST(foo AS CHAR)" - ) - dialect = mysql.dialect() - dialect.server_version_info = (3, 2, 3) - with expect_warnings("Current MySQL version does not support CAST;"): - self.assert_compile( - cast(Column("foo", Integer), String), "foo", dialect=dialect - ) - def test_cast_grouped_expression_non_castable(self): with expect_warnings("Datatype FLOAT does not support CAST on MySQL;"): self.assert_compile( cast(sql.column("x") + sql.column("y"), Float), "(x + y)" ) - def test_cast_grouped_expression_pre_4(self): - dialect = mysql.dialect() - dialect.server_version_info = (3, 2, 3) - with expect_warnings("Current MySQL version does not support CAST;"): - self.assert_compile( - cast(sql.column("x") + sql.column("y"), Integer), - "(x + y)", - dialect=dialect, - ) - def test_extract(self): t = sql.table("t", sql.column("col1")) diff --git a/test/dialect/mysql/test_dialect.py b/test/dialect/mysql/test_dialect.py index 41a4af6395..c3bd94ffad 100644 --- a/test/dialect/mysql/test_dialect.py +++ b/test/dialect/mysql/test_dialect.py @@ -25,7 +25,7 @@ from ...engine import test_execute class BackendDialectTest(fixtures.TestBase): __backend__ = True - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" def test_no_show_variables(self): from sqlalchemy.testing import mock @@ -227,7 +227,7 @@ class DialectTest(fixtures.TestBase): )[1] assert "raise_on_warnings" not in kw - @testing.only_on("mysql") + @testing.only_on(["mysql", "mariadb"]) def test_random_arg(self): dialect = testing.db.dialect kw = dialect.create_connect_args( @@ -235,7 +235,7 @@ class DialectTest(fixtures.TestBase): )[1] eq_(kw["foo"], "true") - @testing.only_on("mysql") + @testing.only_on(["mysql", "mariadb"]) @testing.skip_if("mysql+mysqlconnector", "totally broken for the moment") @testing.fails_on("mysql+oursql", "unsupported") def test_special_encodings(self): @@ -272,27 +272,33 @@ class ParseVersionTest(fixtures.TestBase): "5.7.20", ) + def test_502_minimum(self): + dialect = mysql.dialect() + assert_raises_message( + NotImplementedError, + "the MySQL/MariaDB dialect supports server " + "version info 5.0.2 and above.", + dialect._parse_server_version, + "5.0.1", + ) + @testing.combinations( - ((10, 2, 7), "10.2.7-MariaDB", (10, 2, 7, "MariaDB"), True), - ( - (10, 2, 7), - "5.6.15.10.2.7-MariaDB", - (5, 6, 15, 10, 2, 7, "MariaDB"), - True, - ), - ((10, 2, 10), "10.2.10-MariaDB", (10, 2, 10, "MariaDB"), True), + ((10, 2, 7), "10.2.7-MariaDB", (10, 2, 7), True), + ((10, 2, 7), "5.6.15.10.2.7-MariaDB", (5, 6, 15, 10, 2, 7), True,), + ((5, 0, 51, 24), "5.0.51a.24+lenny5", (5, 0, 51, 24), False), + ((10, 2, 10), "10.2.10-MariaDB", (10, 2, 10), True), ((5, 7, 20), "5.7.20", (5, 7, 20), False), ((5, 6, 15), "5.6.15", (5, 6, 15), False), ( (10, 2, 6), "10.2.6.MariaDB.10.2.6+maria~stretch-log", - (10, 2, 6, "MariaDB", 10, 2, "6+maria~stretch", "log"), + (10, 2, 6, 10, 2, 6), True, ), ( (10, 1, 9), "10.1.9-MariaDBV1.0R050D002-20170809-1522", - (10, 1, 9, "MariaDB", "V1", "0R050D002", 20170809, 1522), + (10, 1, 9, 20170809, 1522), True, ), ) @@ -306,16 +312,16 @@ class ParseVersionTest(fixtures.TestBase): assert dialect._is_mariadb is is_mariadb @testing.combinations( - (True, (10, 2, 7, "MariaDB")), - (True, (5, 6, 15, 10, 2, 7, "MariaDB")), - (False, (10, 2, 10, "MariaDB")), - (False, (5, 7, 20)), - (False, (5, 6, 15)), - (True, (10, 2, 6, "MariaDB", 10, 2, "6+maria~stretch", "log")), + (True, "10.2.7-MariaDB"), + (True, "5.6.15-10.2.7-MariaDB"), + (False, "10.2.10-MariaDB"), + (False, "5.7.20"), + (False, "5.6.15"), + (True, "10.2.6-MariaDB-10.2.6+maria~stretch.log"), ) def test_mariadb_check_warning(self, expect_, version): dialect = mysql.dialect(is_mariadb="MariaDB" in version) - dialect.server_version_info = version + dialect._parse_server_version(version) if expect_: with expect_warnings( ".*before 10.2.9 has known issues regarding " @@ -337,7 +343,7 @@ class RemoveUTCTimestampTest(fixtures.TablesTest): """ - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @classmethod @@ -412,7 +418,7 @@ class RemoveUTCTimestampTest(fixtures.TablesTest): class SQLModeDetectionTest(fixtures.TestBase): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True def _options(self, modes): @@ -462,7 +468,7 @@ class SQLModeDetectionTest(fixtures.TestBase): class ExecutionTest(fixtures.TestBase): """Various MySQL execution special cases.""" - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True def test_charset_caching(self): @@ -482,7 +488,7 @@ class ExecutionTest(fixtures.TestBase): class AutocommitTextTest(test_execute.AutocommitTextTest): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" def test_load_data(self): self._test_keyword("LOAD DATA STUFF") diff --git a/test/dialect/mysql/test_for_update.py b/test/dialect/mysql/test_for_update.py index 2c247a5c09..e39a3fcc00 100644 --- a/test/dialect/mysql/test_for_update.py +++ b/test/dialect/mysql/test_for_update.py @@ -23,7 +23,7 @@ from sqlalchemy.testing import fixtures class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): __backend__ = True - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __requires__ = ("mysql_for_update",) @classmethod @@ -36,7 +36,10 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): x = Column(Integer) y = Column(Integer) bs = relationship("B") - __table_args__ = {"mysql_engine": "InnoDB"} + __table_args__ = { + "mysql_engine": "InnoDB", + "mariadb_engine": "InnoDB", + } class B(Base): __tablename__ = "b" @@ -44,7 +47,10 @@ class MySQLForUpdateLockingTest(fixtures.DeclarativeMappedTest): a_id = Column(ForeignKey("a.id")) x = Column(Integer) y = Column(Integer) - __table_args__ = {"mysql_engine": "InnoDB"} + __table_args__ = { + "mysql_engine": "InnoDB", + "mariadb_engine": "InnoDB", + } @classmethod def insert_data(cls, connection): diff --git a/test/dialect/mysql/test_on_duplicate.py b/test/dialect/mysql/test_on_duplicate.py index 45f679a17d..95aabc7769 100644 --- a/test/dialect/mysql/test_on_duplicate.py +++ b/test/dialect/mysql/test_on_duplicate.py @@ -13,7 +13,7 @@ from sqlalchemy.testing.assertions import eq_ class OnDuplicateTest(fixtures.TablesTest): - __only_on__ = ("mysql",) + __only_on__ = ("mysql", "mariadb") __backend__ = True run_define_tables = "each" diff --git a/test/dialect/mysql/test_query.py b/test/dialect/mysql/test_query.py index 4747a1dfe8..a8b80d91cd 100644 --- a/test/dialect/mysql/test_query.py +++ b/test/dialect/mysql/test_query.py @@ -22,7 +22,7 @@ from sqlalchemy.testing import is_ class IdiosyncrasyTest(fixtures.TestBase): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @testing.emits_warning() @@ -44,7 +44,7 @@ class IdiosyncrasyTest(fixtures.TestBase): class MatchTest(fixtures.TestBase): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @classmethod @@ -58,6 +58,7 @@ class MatchTest(fixtures.TestBase): Column("id", Integer, primary_key=True), Column("description", String(50)), mysql_engine="MyISAM", + mariadb_engine="MyISAM", ) matchtable = Table( "matchtable", @@ -66,6 +67,7 @@ class MatchTest(fixtures.TestBase): Column("title", String(200)), Column("category_id", Integer, ForeignKey("cattable.id")), mysql_engine="MyISAM", + mariadb_engine="MyISAM", ) metadata.create_all() @@ -216,7 +218,7 @@ class MatchTest(fixtures.TestBase): class AnyAllTest(fixtures.TablesTest): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @classmethod diff --git a/test/dialect/mysql/test_reflection.py b/test/dialect/mysql/test_reflection.py index f0465ec50c..026025a88f 100644 --- a/test/dialect/mysql/test_reflection.py +++ b/test/dialect/mysql/test_reflection.py @@ -41,7 +41,7 @@ from sqlalchemy.testing import mock class TypeReflectionTest(fixtures.TestBase): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @testing.provide_metadata @@ -228,7 +228,7 @@ class TypeReflectionTest(fixtures.TestBase): class ReflectionTest(fixtures.TestBase, AssertsCompiledSQL): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True def test_default_reflection(self): @@ -304,18 +304,31 @@ class ReflectionTest(fixtures.TestBase, AssertsCompiledSQL): def test_reflection_with_table_options(self): comment = r"""Comment types type speedily ' " \ '' Fun!""" + if testing.against("mariadb"): + kwargs = dict( + mariadb_engine="MEMORY", + mariadb_default_charset="utf8", + mariadb_auto_increment="5", + mariadb_avg_row_length="3", + mariadb_password="secret", + mariadb_connection="fish", + ) + else: + kwargs = dict( + mysql_engine="MEMORY", + mysql_default_charset="utf8", + mysql_auto_increment="5", + mysql_avg_row_length="3", + mysql_password="secret", + mysql_connection="fish", + ) def_table = Table( "mysql_def", MetaData(testing.db), Column("c1", Integer()), - mysql_engine="MEMORY", comment=comment, - mysql_default_charset="utf8", - mysql_auto_increment="5", - mysql_avg_row_length="3", - mysql_password="secret", - mysql_connection="fish", + **kwargs ) def_table.create() @@ -324,27 +337,50 @@ class ReflectionTest(fixtures.TestBase, AssertsCompiledSQL): finally: def_table.drop() - assert def_table.kwargs["mysql_engine"] == "MEMORY" - assert def_table.comment == comment - assert def_table.kwargs["mysql_default_charset"] == "utf8" - assert def_table.kwargs["mysql_auto_increment"] == "5" - assert def_table.kwargs["mysql_avg_row_length"] == "3" - assert def_table.kwargs["mysql_password"] == "secret" - assert def_table.kwargs["mysql_connection"] == "fish" + if testing.against("mariadb"): + assert def_table.kwargs["mariadb_engine"] == "MEMORY" + assert def_table.comment == comment + assert def_table.kwargs["mariadb_default_charset"] == "utf8" + assert def_table.kwargs["mariadb_auto_increment"] == "5" + assert def_table.kwargs["mariadb_avg_row_length"] == "3" + assert def_table.kwargs["mariadb_password"] == "secret" + assert def_table.kwargs["mariadb_connection"] == "fish" + + assert reflected.kwargs["mariadb_engine"] == "MEMORY" + + assert reflected.comment == comment + assert reflected.kwargs["mariadb_comment"] == comment + assert reflected.kwargs["mariadb_default charset"] == "utf8" + assert reflected.kwargs["mariadb_avg_row_length"] == "3" + assert reflected.kwargs["mariadb_connection"] == "fish" + + # This field doesn't seem to be returned by mariadb itself. + # assert reflected.kwargs['mariadb_password'] == 'secret' + + # This is explicitly ignored when reflecting schema. + # assert reflected.kwargs['mariadb_auto_increment'] == '5' + else: + assert def_table.kwargs["mysql_engine"] == "MEMORY" + assert def_table.comment == comment + assert def_table.kwargs["mysql_default_charset"] == "utf8" + assert def_table.kwargs["mysql_auto_increment"] == "5" + assert def_table.kwargs["mysql_avg_row_length"] == "3" + assert def_table.kwargs["mysql_password"] == "secret" + assert def_table.kwargs["mysql_connection"] == "fish" - assert reflected.kwargs["mysql_engine"] == "MEMORY" + assert reflected.kwargs["mysql_engine"] == "MEMORY" - assert reflected.comment == comment - assert reflected.kwargs["mysql_comment"] == comment - assert reflected.kwargs["mysql_default charset"] == "utf8" - assert reflected.kwargs["mysql_avg_row_length"] == "3" - assert reflected.kwargs["mysql_connection"] == "fish" + assert reflected.comment == comment + assert reflected.kwargs["mysql_comment"] == comment + assert reflected.kwargs["mysql_default charset"] == "utf8" + assert reflected.kwargs["mysql_avg_row_length"] == "3" + assert reflected.kwargs["mysql_connection"] == "fish" - # This field doesn't seem to be returned by mysql itself. - # assert reflected.kwargs['mysql_password'] == 'secret' + # This field doesn't seem to be returned by mysql itself. + # assert reflected.kwargs['mysql_password'] == 'secret' - # This is explicitly ignored when reflecting schema. - # assert reflected.kwargs['mysql_auto_increment'] == '5' + # This is explicitly ignored when reflecting schema. + # assert reflected.kwargs['mysql_auto_increment'] == '5' def test_reflection_on_include_columns(self): """Test reflection of include_columns to be sure they respect case.""" @@ -714,15 +750,22 @@ class ReflectionTest(fixtures.TestBase, AssertsCompiledSQL): self.metadata, Column("id", Integer, primary_key=True), Column("textdata", String(50)), + mariadb_engine="InnoDB", mysql_engine="InnoDB", ) - Index("textdata_ix", mt.c.textdata, mysql_prefix="FULLTEXT") + + Index( + "textdata_ix", + mt.c.textdata, + mysql_prefix="FULLTEXT", + mariadb_prefix="FULLTEXT", + ) self.metadata.create_all(testing.db) mt = Table("mytable", MetaData(), autoload_with=testing.db) idx = list(mt.indexes)[0] eq_(idx.name, "textdata_ix") - eq_(idx.dialect_options["mysql"]["prefix"], "FULLTEXT") + eq_(idx.dialect_options[testing.db.name]["prefix"], "FULLTEXT") self.assert_compile( CreateIndex(idx), "CREATE FULLTEXT INDEX textdata_ix ON mytable (textdata)", diff --git a/test/dialect/mysql/test_types.py b/test/dialect/mysql/test_types.py index d9afb0063a..bc5619cdbe 100644 --- a/test/dialect/mysql/test_types.py +++ b/test/dialect/mysql/test_types.py @@ -469,7 +469,7 @@ class TypeCompileTest(fixtures.TestBase, AssertsCompiledSQL): class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults): __dialect__ = mysql.dialect() - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True # fixed in mysql-connector as of 2.0.1, @@ -755,7 +755,7 @@ class TypeRoundTripTest(fixtures.TestBase, AssertsExecutionResults): class JSONTest(fixtures.TestBase): __requires__ = ("json_type",) - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __backend__ = True @testing.provide_metadata @@ -791,7 +791,7 @@ class EnumSetTest( fixtures.TestBase, AssertsExecutionResults, AssertsCompiledSQL ): - __only_on__ = "mysql" + __only_on__ = "mysql", "mariadb" __dialect__ = mysql.dialect() __backend__ = True diff --git a/test/dialect/postgresql/test_dialect.py b/test/dialect/postgresql/test_dialect.py index 57c243442e..d15e3a843c 100644 --- a/test/dialect/postgresql/test_dialect.py +++ b/test/dialect/postgresql/test_dialect.py @@ -828,8 +828,9 @@ $$ LANGUAGE plpgsql; def test_extract(self, connection): fivedaysago = testing.db.scalar( - select(func.now()) + select(func.now().op("at time zone")("UTC")) ) - datetime.timedelta(days=5) + for field, exp in ( ("year", fivedaysago.year), ("month", fivedaysago.month), @@ -837,7 +838,11 @@ $$ LANGUAGE plpgsql; ): r = connection.execute( select( - extract(field, func.now() + datetime.timedelta(days=-5)) + extract( + field, + func.now().op("at time zone")("UTC") + + datetime.timedelta(days=-5), + ) ) ).scalar() eq_(r, exp) diff --git a/test/orm/test_naturalpks.py b/test/orm/test_naturalpks.py index c13f56b857..202ff9ab01 100644 --- a/test/orm/test_naturalpks.py +++ b/test/orm/test_naturalpks.py @@ -871,7 +871,7 @@ class ReversePKsTest(fixtures.MappedTest): class SelfReferentialTest(fixtures.MappedTest): # mssql, mysql don't allow # ON UPDATE on self-referential keys - __unsupported_on__ = ("mssql", "mysql") + __unsupported_on__ = ("mssql", "mysql", "mariadb") __requires__ = ("on_update_or_deferrable_fks",) __backend__ = True diff --git a/test/orm/test_query.py b/test/orm/test_query.py index 99389439ea..e43504d9e2 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -2654,7 +2654,9 @@ class FilterTest(QueryTest, AssertsCompiledSQL): [User(id=8), User(id=9)], ) - @testing.fails_on("mysql", "doesn't like CAST in the limit clause") + @testing.fails_on( + ["mysql", "mariadb"], "doesn't like CAST in the limit clause" + ) @testing.requires.bound_limit_offset def test_select_with_bindparam_offset_limit_w_cast(self): User = self.classes.User diff --git a/test/requirements.py b/test/requirements.py index fdb7c2ff33..99a3605658 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -6,7 +6,6 @@ import sys from sqlalchemy import exc -from sqlalchemy import util from sqlalchemy.sql import text from sqlalchemy.testing import exclusions from sqlalchemy.testing.exclusions import against @@ -40,6 +39,7 @@ class DefaultRequirements(SuiteRequirements): [ no_support("firebird", "not supported by database"), no_support("mysql", "not supported by database"), + no_support("mariadb", "not supported by database"), no_support("mssql", "not supported by database"), ] ) @@ -111,7 +111,7 @@ class DefaultRequirements(SuiteRequirements): @property def foreign_key_constraint_option_reflection_ondelete(self): - return only_on(["postgresql", "mysql", "sqlite", "oracle"]) + return only_on(["postgresql", "mysql", "mariadb", "sqlite", "oracle"]) @property def fk_constraint_option_reflection_ondelete_restrict(self): @@ -119,11 +119,11 @@ class DefaultRequirements(SuiteRequirements): @property def fk_constraint_option_reflection_ondelete_noaction(self): - return only_on(["postgresql", "mysql", "sqlite"]) + return only_on(["postgresql", "mysql", "mariadb", "sqlite"]) @property def foreign_key_constraint_option_reflection_onupdate(self): - return only_on(["postgresql", "mysql", "sqlite"]) + return only_on(["postgresql", "mysql", "mariadb", "sqlite"]) @property def fk_constraint_option_reflection_onupdate_restrict(self): @@ -131,14 +131,15 @@ class DefaultRequirements(SuiteRequirements): @property def comment_reflection(self): - return only_on(["postgresql", "mysql", "oracle"]) + return only_on(["postgresql", "mysql", "mariadb", "oracle"]) @property def unbounded_varchar(self): """Target database must support VARCHAR with no length""" return skip_if( - ["firebird", "oracle", "mysql"], "not supported by database" + ["firebird", "oracle", "mysql", "mariadb"], + "not supported by database", ) @property @@ -183,7 +184,14 @@ class DefaultRequirements(SuiteRequirements): @property def qmark_paramstyle(self): return only_on( - ["firebird", "sqlite", "+pyodbc", "+mxodbc", "mysql+oursql"] + [ + "firebird", + "sqlite", + "+pyodbc", + "+mxodbc", + "mysql+oursql", + "mariadb+oursql", + ] ) @property @@ -198,6 +206,10 @@ class DefaultRequirements(SuiteRequirements): "mysql+pymysql", "mysql+cymysql", "mysql+mysqlconnector", + "mariadb+mysqldb", + "mariadb+pymysql", + "mariadb+cymysql", + "mariadb+mysqlconnector", "postgresql+pg8000", ] ) @@ -213,6 +225,9 @@ class DefaultRequirements(SuiteRequirements): "mysql+mysqlconnector", "mysql+pymysql", "mysql+cymysql", + "mariadb+mysqlconnector", + "mariadb+pymysql", + "mariadb+cymysql", "mssql+pymssql", ] ) @@ -310,14 +325,14 @@ class DefaultRequirements(SuiteRequirements): config, "sqlite" ) and config.db.dialect.dbapi.sqlite_version_info >= (3, 15, 0) - return only_on(["mysql", "postgresql", _sqlite_tuple_in]) + return only_on(["mysql", "mariadb", "postgresql", _sqlite_tuple_in]) @property def independent_cursors(self): """Target must support simultaneous, independent database cursors on a single connection.""" - return skip_if(["mssql", "mysql"], "no driver support") + return skip_if(["mssql", "mysql", "mariadb"], "no driver support") @property def independent_connections(self): @@ -369,7 +384,7 @@ class DefaultRequirements(SuiteRequirements): @property def isolation_level(self): return only_on( - ("postgresql", "sqlite", "mysql", "mssql", "oracle"), + ("postgresql", "sqlite", "mysql", "mariadb", "mssql", "oracle"), "DBAPI has no isolation level support", ) + fails_on( "postgresql+pypostgresql", @@ -388,6 +403,9 @@ class DefaultRequirements(SuiteRequirements): elif against(config, "mysql"): default = "REPEATABLE READ" levels.add("AUTOCOMMIT") + elif against(config, "mariadb"): + default = "REPEATABLE READ" + levels.add("AUTOCOMMIT") elif against(config, "mssql"): default = "READ COMMITTED" levels.add("AUTOCOMMIT") @@ -416,6 +434,7 @@ class DefaultRequirements(SuiteRequirements): [ # no access to same table no_support("mysql", "requires SUPER priv"), + no_support("mariadb", "requires SUPER priv"), exclude("mysql", "<", (5, 0, 10), "not supported by database"), # huh? TODO: implement triggers for PG tests, remove this no_support( @@ -457,7 +476,7 @@ class DefaultRequirements(SuiteRequirements): """Target must support UPDATE..FROM syntax""" return only_on( - ["postgresql", "mssql", "mysql"], + ["postgresql", "mssql", "mysql", "mariadb"], "Backend does not support UPDATE..FROM", ) @@ -465,7 +484,7 @@ class DefaultRequirements(SuiteRequirements): def delete_from(self): """Target must support DELETE FROM..FROM or DELETE..USING syntax""" return only_on( - ["postgresql", "mssql", "mysql", "sybase"], + ["postgresql", "mssql", "mysql", "mariadb", "sybase"], "Backend does not support DELETE..FROM", ) @@ -516,7 +535,7 @@ class DefaultRequirements(SuiteRequirements): def cross_schema_fk_reflection(self): """target system must support reflection of inter-schema foreign keys """ - return only_on(["postgresql", "mysql", "mssql"]) + return only_on(["postgresql", "mysql", "mariadb", "mssql"]) @property def implicit_default_schema(self): @@ -531,7 +550,7 @@ class DefaultRequirements(SuiteRequirements): @property def unique_constraint_reflection(self): return fails_on_everything_except( - "postgresql", "mysql", "sqlite", "oracle" + "postgresql", "mysql", "mariadb", "sqlite", "oracle" ) @property @@ -539,6 +558,7 @@ class DefaultRequirements(SuiteRequirements): return ( self.unique_constraint_reflection + skip_if("mysql") + + skip_if("mariadb") + skip_if("oracle") ) @@ -572,15 +592,14 @@ class DefaultRequirements(SuiteRequirements): def update_nowait(self): """Target database must support SELECT...FOR UPDATE NOWAIT""" return skip_if( - ["firebird", "mssql", "mysql", "sqlite", "sybase"], + ["firebird", "mssql", "mysql", "mariadb", "sqlite", "sybase"], "no FOR UPDATE NOWAIT support", ) @property def subqueries(self): """Target database must support subqueries.""" - - return skip_if(exclude("mysql", "<", (4, 1, 1)), "no subquery support") + return exclusions.open() @property def ctes(self): @@ -599,6 +618,7 @@ class DefaultRequirements(SuiteRequirements): and config.db.dialect.server_version_info >= (8,) ) ), + "mariadb>10.2", "postgresql", "mssql", "oracle", @@ -633,7 +653,9 @@ class DefaultRequirements(SuiteRequirements): """target database must use a plain percent '%' as the 'modulus' operator.""" - return only_if(["mysql", "sqlite", "postgresql+psycopg2", "mssql"]) + return only_if( + ["mysql", "mariadb", "sqlite", "postgresql+psycopg2", "mssql"] + ) @property def intersect(self): @@ -700,7 +722,7 @@ class DefaultRequirements(SuiteRequirements): def sql_expression_limit_offset(self): return ( fails_if( - ["mysql"], + ["mysql", "mariadb"], "Target backend can't accommodate full expressions in " "OFFSET or LIMIT", ) @@ -769,7 +791,8 @@ class DefaultRequirements(SuiteRequirements): def two_phase_recovery(self): return self.two_phase_transactions + ( skip_if( - "mysql", "still can't get recover to work w/ MariaDB / MySQL" + ["mysql", "mariadb"], + "still can't get recover to work w/ MariaDB / MySQL", ) ) @@ -817,31 +840,19 @@ class DefaultRequirements(SuiteRequirements): def unicode_connections(self): """ Target driver must support some encoding of Unicode across the wire. + """ - # TODO: expand to exclude MySQLdb versions w/ broken unicode - return skip_if( - [exclude("mysql", "<", (4, 1, 1), "no unicode connection support")] - ) + return exclusions.open() @property def unicode_ddl(self): """Target driver must support some degree of non-ascii symbol names.""" - # TODO: expand to exclude MySQLdb versions w/ broken unicode return skip_if( [ no_support("oracle", "FIXME: no support in database?"), no_support("sybase", "FIXME: guessing, needs confirmation"), no_support("mssql+pymssql", "no FreeTDS support"), - LambdaPredicate( - lambda config: against(config, "mysql+mysqlconnector") - and config.db.dialect._mysqlconnector_version_info > (2, 0) - and util.py2k, - "bug in mysqlconnector 2.0", - ), - exclude( - "mysql", "<", (4, 1, 1), "no unicode connection support" - ), ] ) @@ -861,7 +872,26 @@ class DefaultRequirements(SuiteRequirements): after an insert() construct executes. """ return fails_on_everything_except( - "mysql", "sqlite+pysqlite", "sqlite+pysqlcipher", "sybase", "mssql" + "mysql", + "mariadb", + "sqlite+pysqlite", + "sqlite+pysqlcipher", + "sybase", + "mssql", + ) + + @property + def emulated_lastrowid_even_with_sequences(self): + """"target dialect retrieves cursor.lastrowid or an equivalent + after an insert() construct executes, even if the table has a + Sequence on it.. + """ + return fails_on_everything_except( + "mysql", + "mariadb", + "sqlite+pysqlite", + "sqlite+pysqlcipher", + "sybase", ) @property @@ -877,7 +907,11 @@ class DefaultRequirements(SuiteRequirements): return skip_if( "mssql+pymssql", "crashes on pymssql" ) + fails_on_everything_except( - "mysql", "sqlite+pysqlite", "sqlite+pysqlcipher", "mssql" + "mysql", + "mariadb", + "sqlite+pysqlite", + "sqlite+pysqlcipher", + "mssql", ) @property @@ -926,6 +960,7 @@ class DefaultRequirements(SuiteRequirements): >= (10, 2, 7) ) ), + "mariadb>=10.2.7", "postgresql >= 9.3", self._sqlite_json, ] @@ -938,6 +973,7 @@ class DefaultRequirements(SuiteRequirements): [ lambda config: against(config, "mysql") and config.db.dialect._is_mariadb, + "mariadb", "sqlite", ] ) @@ -1004,7 +1040,9 @@ class DefaultRequirements(SuiteRequirements): """target dialect supports representation of Python datetime.datetime() with microsecond objects.""" - return skip_if(["mssql", "mysql", "firebird", "oracle", "sybase"]) + return skip_if( + ["mssql", "mysql", "mariadb", "firebird", "oracle", "sybase"] + ) @property def timestamp_microseconds(self): @@ -1055,7 +1093,9 @@ class DefaultRequirements(SuiteRequirements): """target dialect supports representation of Python datetime.time() with microsecond objects.""" - return skip_if(["mssql", "mysql", "firebird", "oracle", "sybase"]) + return skip_if( + ["mssql", "mysql", "mariadb", "firebird", "oracle", "sybase"] + ) @property def precision_numerics_general(self): @@ -1141,6 +1181,12 @@ class DefaultRequirements(SuiteRequirements): None, "mysql FLOAT type only returns 4 decimals", ), + ( + "mariadb", + None, + None, + "mysql FLOAT type only returns 4 decimals", + ), ( "firebird", None, @@ -1155,6 +1201,7 @@ class DefaultRequirements(SuiteRequirements): return fails_if( [ ("mysql+oursql", None, None, "Floating point error"), + ("mariadb+oursql", None, None, "Floating point error"), ( "firebird", None, @@ -1198,24 +1245,7 @@ class DefaultRequirements(SuiteRequirements): """ - # fixed for mysqlclient in - # https://github.com/PyMySQL/mysqlclient-python/commit/68b9662918577fc05be9610ef4824a00f2b051b0 - def check(config): - if against(config, "mysql+mysqldb"): - # can remove once post 1.3.13 is released - try: - from MySQLdb import converters - from decimal import Decimal - - return Decimal not in converters.conversions - except: - return True - - return against( - config, "mysql+mysqldb" - ) and config.db.dialect._mysql_dbapi_version <= (1, 3, 13) - - return exclusions.fails_on(check, "fixed for mysqlclient post 1.3.13") + return exclusions.open() @property def fetch_null_from_numeric(self): @@ -1334,6 +1364,7 @@ class DefaultRequirements(SuiteRequirements): "bind placeholders", ), ("mysql", None, None, "executemany() doesn't work here"), + ("mariadb", None, None, "executemany() doesn't work here"), ] ) @@ -1360,6 +1391,7 @@ class DefaultRequirements(SuiteRequirements): # note MySQL databases need to be created w/ utf8mb4 charset # for the test suite "mysql": "utf8mb4_bin", + "mariadb": "utf8mb4_bin", "sqlite": "NOCASE", # will raise *with* quoting "mssql": "Latin1_General_CI_AS", @@ -1452,7 +1484,7 @@ class DefaultRequirements(SuiteRequirements): @property def mysql_fsp(self): - return only_if("mysql >= 5.6.4") + return only_if(["mysql >= 5.6.4", "mariadb"]) @property def mysql_fully_case_sensitive(self): @@ -1516,7 +1548,7 @@ class DefaultRequirements(SuiteRequirements): def _mysql_and_check_constraints_exist(self, config): # 1. we have mysql / mariadb and # 2. it enforces check constraints - if exclusions.against(config, "mysql"): + if exclusions.against(config, ["mysql", "mariadb"]): if config.db.dialect._is_mariadb: norm_version_info = ( config.db.dialect._mariadb_normalized_version_info @@ -1532,7 +1564,7 @@ class DefaultRequirements(SuiteRequirements): # 1. we dont have mysql / mariadb or # 2. we have mysql / mariadb that enforces check constraints return not exclusions.against( - config, "mysql" + config, ["mysql", "mariadb"] ) or self._mysql_and_check_constraints_exist(config) def _mysql_check_constraints_dont_exist(self, config): @@ -1541,28 +1573,27 @@ class DefaultRequirements(SuiteRequirements): return not self._mysql_check_constraints_exist(config) def _mysql_not_mariadb_102(self, config): - return against(config, "mysql") and ( + return (against(config, ["mysql", "mariadb"])) and ( not config.db.dialect._is_mariadb or config.db.dialect._mariadb_normalized_version_info < (10, 2) ) def _mysql_not_mariadb_103(self, config): - return against(config, "mysql") and ( + return (against(config, ["mysql", "mariadb"])) and ( not config.db.dialect._is_mariadb or config.db.dialect._mariadb_normalized_version_info < (10, 3) ) def _mysql_not_mariadb_104(self, config): - return against(config, "mysql") and ( + return (against(config, ["mysql", "mariadb"])) and ( not config.db.dialect._is_mariadb or config.db.dialect._mariadb_normalized_version_info < (10, 4) ) def _has_mysql_on_windows(self, config): return ( - against(config, "mysql") - and config.db.dialect._detect_casing(config.db) == 1 - ) + against(config, ["mysql", "mariadb"]) + ) and config.db.dialect._detect_casing(config.db) == 1 def _has_mysql_fully_case_sensitive(self, config): return ( diff --git a/test/sql/test_insert_exec.py b/test/sql/test_insert_exec.py index becca12ffd..e27decd6f1 100644 --- a/test/sql/test_insert_exec.py +++ b/test/sql/test_insert_exec.py @@ -274,6 +274,7 @@ class InsertExecTest(fixtures.TablesTest): metadata, Column("id", Integer, primary_key=True), mysql_engine="MyISAM", + mariadb_engine="MyISAM", ) t6 = Table( "t6", @@ -291,6 +292,7 @@ class InsertExecTest(fixtures.TablesTest): test_needs_autoincrement=True, ), mysql_engine="MyISAM", + mariadb_engine="MyISAM", ) metadata.create_all() diff --git a/test/sql/test_types.py b/test/sql/test_types.py index fac9fd1399..0895aeda0a 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -3139,7 +3139,7 @@ class BooleanTest( ) @testing.fails_on( - "mysql", + ["mysql", "mariadb"], "The CHECK clause is parsed but ignored by all storage engines.", ) @testing.fails_on("mssql", "FIXME: MS-SQL 2005 doesn't honor CHECK ?!?")