From: Mike Bayer Date: Mon, 4 Jul 2016 19:54:29 +0000 (-0400) Subject: Double percent signs based on paramstyle, not dialect X-Git-Tag: rel_1_2_0b1~111^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=86ef507cc73ee4a0a104b334d7ce08ad045e0c76;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Double percent signs based on paramstyle, not dialect This patch moves the "doubling" of percent signs into the base compiler and makes it completely a product of whether or not the paramstyle is format/pyformat or not. Without this paramstyle, percent signs are not doubled across text(), literal_column(), and column(). Change-Id: Ie2f278ab1dbb94b5078f85c0096d74dbfa049197 Fixes: #3740 --- diff --git a/doc/build/changelog/changelog_12.rst b/doc/build/changelog/changelog_12.rst index ad9e73160f..80be30df26 100644 --- a/doc/build/changelog/changelog_12.rst +++ b/doc/build/changelog/changelog_12.rst @@ -13,6 +13,24 @@ .. changelog:: :version: 1.2.0b1 + .. change:: 3740 + :tags: bug, sql + :tickets: 3740 + + The system by which percent signs in SQL statements are "doubled" + for escaping purposes has been refined. The "doubling" of percent + signs mostly associated with the :obj:`.literal_column` construct + as well as operators like :meth:`.ColumnOperators.contains` now + occurs based on the stated paramstyle of the DBAPI in use; for + percent-sensitive paramstyles as are common with the Postgresql + and MySQL drivers the doubling will occur, for others like that + of SQLite it will not. This allows more database-agnostic use + of the :obj:`.literal_column` construct to be possible. + + .. seealso:: + + :ref:`change_3740` + .. change:: 3957 :tags: bug, sql :tickets: 3957 diff --git a/doc/build/changelog/migration_12.rst b/doc/build/changelog/migration_12.rst index 4c2090e50e..ae4dccd5f3 100644 --- a/doc/build/changelog/migration_12.rst +++ b/doc/build/changelog/migration_12.rst @@ -362,6 +362,41 @@ Where the value of the parameter "x_1" is ``'total/%score'``. Key Behavioral Changes - ORM ============================ +.. _change_3740: + +Percent signs in literal_column() now conditionally escaped +------------------------------------------------------------ + +The :obj:`.literal_column` construct now escapes percent sign characters +conditionally, based on whether or not the DBAPI in use makes use of a +percent-sign-sensitive paramstyle or not (e.g. 'format' or 'pyformat'). + +Previously, it was not possible to produce a :obj:`.literal_column` +construct that stated a single percent sign:: + + >>> from sqlalchemy import literal_column + >>> print(literal_column('some%symbol')) + some%%symbol + +The percent sign is now unaffected for dialects that are not set to +use the 'format' or 'pyformat' paramstyles; dialects such most MySQL +dialects which do state one of these paramstyles will continue to escape +as is appropriate:: + + >>> from sqlalchemy import literal_column + >>> print(literal_column('some%symbol')) + some%symbol + >>> from sqlalchemy.dialects import mysql + >>> print(literal_column('some%symbol').compile(dialect=mysql.dialect())) + some%%symbol + +As part of this change, the doubling that has been present when using +operators like :meth:`.ColumnOperators.contains`, +:meth:`.ColumnOperators.startswith` and :meth:`.ColumnOperators.endswith` +is also refined to only occur when appropriate. + +:ticket:`3740` + .. _change_3934: The after_rollback() Session event now emits before the expiration of objects diff --git a/doc/build/core/tutorial.rst b/doc/build/core/tutorial.rst index 2e47aa324b..f368c72379 100644 --- a/doc/build/core/tutorial.rst +++ b/doc/build/core/tutorial.rst @@ -1857,7 +1857,7 @@ is the DISTINCT modifier. A simple DISTINCT clause can be added using the >>> conn.execute(stmt).fetchall() {opensql}SELECT DISTINCT users.name FROM users, addresses - WHERE (addresses.email_address LIKE '%%' || users.name || '%%') + WHERE (addresses.email_address LIKE '%' || users.name || '%') () {stop}[(u'jack',), (u'wendy',)] @@ -2019,7 +2019,7 @@ The resulting SQL from the above statement would render as:: UPDATE users SET name=:name FROM addresses WHERE users.id = addresses.id AND - addresses.email_address LIKE :email_address_1 || '%%' + addresses.email_address LIKE :email_address_1 || '%' When using MySQL, columns from each table can be assigned to in the SET clause directly, using the dictionary form passed to :meth:`.Update.values`:: @@ -2036,7 +2036,7 @@ The tables are referenced explicitly in the SET clause:: UPDATE users, addresses SET addresses.email_address=%s, users.name=%s WHERE users.id = addresses.id - AND addresses.email_address LIKE concat(%s, '%%') + AND addresses.email_address LIKE concat(%s, '%') SQLAlchemy doesn't do anything special when these constructs are used on a non-supporting database. The ``UPDATE FROM`` syntax generates by default diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 6af860133f..7941d4c418 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -64,19 +64,11 @@ class MySQLExecutionContext_mysqldb(MySQLExecutionContext): class MySQLCompiler_mysqldb(MySQLCompiler): - def visit_mod_binary(self, binary, operator, **kw): - return self.process(binary.left, **kw) + " %% " + \ - self.process(binary.right, **kw) - - def post_process_text(self, text): - return text.replace('%', '%%') + pass class MySQLIdentifierPreparer_mysqldb(MySQLIdentifierPreparer): - - def _escape_identifier(self, value): - value = value.replace(self.escape_quote, self.escape_to_quote) - return value.replace("%", "%%") + pass class MySQLDialect_mysqldb(MySQLDialect): diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index 50328143e3..31792a4922 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -455,18 +455,11 @@ class PGExecutionContext_psycopg2(PGExecutionContext): class PGCompiler_psycopg2(PGCompiler): - def visit_mod_binary(self, binary, operator, **kw): - return self.process(binary.left, **kw) + " %% " + \ - self.process(binary.right, **kw) - - def post_process_text(self, text): - return text.replace('%', '%%') + pass class PGIdentifierPreparer_psycopg2(PGIdentifierPreparer): - def _escape_identifier(self, value): - value = value.replace(self.escape_quote, self.escape_to_quote) - return value.replace('%', '%%') + pass class PGDialect_psycopg2(PGDialect): diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index b18f90312f..cc42480096 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -695,7 +695,6 @@ class SQLCompiler(Compiled): name = self.escape_literal_column(name) else: name = self.preparer.quote(name) - table = column.table if table is None or not include_table or not table.named_with_column: return name @@ -715,12 +714,6 @@ class SQLCompiler(Compiled): self.preparer.quote(tablename) + \ "." + name - def escape_literal_column(self, text): - """provide escaping for the literal_column() construct.""" - - # TODO: some dialects might need different behavior here - return text.replace('%', '%%') - def visit_fromclause(self, fromclause, **kwargs): return fromclause.name @@ -732,6 +725,13 @@ class SQLCompiler(Compiled): return self.dialect.type_compiler.process(typeclause.type, **kw) def post_process_text(self, text): + if self.preparer._double_percents: + text = text.replace('%', '%%') + return text + + def escape_literal_column(self, text): + if self.preparer._double_percents: + text = text.replace('%', '%%') return text def visit_textclause(self, textclause, **kw): @@ -1048,6 +1048,14 @@ class SQLCompiler(Compiled): else: return self._generate_generic_binary(binary, opstring, **kw) + def visit_mod_binary(self, binary, operator, **kw): + if self.preparer._double_percents: + return self.process(binary.left, **kw) + " %% " + \ + self.process(binary.right, **kw) + else: + return self.process(binary.left, **kw) + " % " + \ + self.process(binary.right, **kw) + def visit_custom_op_binary(self, element, operator, **kw): kw['eager_grouping'] = operator.eager_grouping return self._generate_generic_binary( @@ -2888,6 +2896,7 @@ class IdentifierPreparer(object): self.escape_to_quote = self.escape_quote * 2 self.omit_schema = omit_schema self._strings = {} + self._double_percents = self.dialect.paramstyle in ('format', 'pyformat') def _with_schema_translate(self, schema_translate_map): prep = self.__class__.__new__(self.__class__) @@ -2902,7 +2911,10 @@ class IdentifierPreparer(object): escaping behavior. """ - return value.replace(self.escape_quote, self.escape_to_quote) + value = value.replace(self.escape_quote, self.escape_to_quote) + if self._double_percents: + value = value.replace('%', '%%') + return value def _unescape_identifier(self, value): """Canonicalize an escaped identifier. diff --git a/lib/sqlalchemy/testing/suite/test_dialect.py b/lib/sqlalchemy/testing/suite/test_dialect.py index 00884a212c..0e62c347fe 100644 --- a/lib/sqlalchemy/testing/suite/test_dialect.py +++ b/lib/sqlalchemy/testing/suite/test_dialect.py @@ -1,9 +1,11 @@ from .. import fixtures, config from ..config import requirements from sqlalchemy import exc -from sqlalchemy import Integer, String +from sqlalchemy import Integer, String, select, literal_column from .. import assert_raises from ..schema import Table, Column +from .. import provide_metadata +from .. import eq_ class ExceptionTest(fixtures.TablesTest): @@ -39,3 +41,33 @@ class ExceptionTest(fixtures.TablesTest): self.tables.manual_pk.insert(), {'id': 1, 'data': 'd1'} ) + + +class EscapingTest(fixtures.TestBase): + @provide_metadata + def test_percent_sign_round_trip(self): + """test that the DBAPI accommodates for escaped / nonescaped + percent signs in a way that matches the compiler + + """ + m = self.metadata + t = Table('t', m, Column('data', String(50))) + t.create(config.db) + with config.db.begin() as conn: + conn.execute(t.insert(), dict(data="some % value")) + conn.execute(t.insert(), dict(data="some %% other value")) + + eq_( + conn.scalar( + select([t.c.data]).where( + t.c.data == literal_column("'some % value'")) + ), + "some % value" + ) + + eq_( + conn.scalar( + select([t.c.data]).where( + t.c.data == literal_column("'some %% other value'")) + ), "some %% other value" + ) diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index 7c3ce13898..217af43377 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -2258,56 +2258,56 @@ class ComposedLikeOperatorsTest(fixtures.TestBase, testing.AssertsCompiledSQL): def test_contains(self): self.assert_compile( column('x').contains('y'), - "x LIKE '%%' || :x_1 || '%%'", + "x LIKE '%' || :x_1 || '%'", checkparams={'x_1': 'y'} ) def test_contains_escape(self): self.assert_compile( column('x').contains('a%b_c', escape='\\'), - "x LIKE '%%' || :x_1 || '%%' ESCAPE '\\'", + "x LIKE '%' || :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_contains_autoescape(self): self.assert_compile( column('x').contains('a%b_c', autoescape='\\'), - "x LIKE '%%' || :x_1 || '%%' ESCAPE '\\'", + "x LIKE '%' || :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) def test_contains_literal(self): self.assert_compile( column('x').contains(literal_column('y')), - "x LIKE '%%' || y || '%%'", + "x LIKE '%' || y || '%'", checkparams={} ) def test_contains_text(self): self.assert_compile( column('x').contains(text('y')), - "x LIKE '%%' || y || '%%'", + "x LIKE '%' || y || '%'", checkparams={} ) def test_not_contains(self): self.assert_compile( ~column('x').contains('y'), - "x NOT LIKE '%%' || :x_1 || '%%'", + "x NOT LIKE '%' || :x_1 || '%'", checkparams={'x_1': 'y'} ) def test_not_contains_escape(self): self.assert_compile( ~column('x').contains('a%b_c', escape='\\'), - "x NOT LIKE '%%' || :x_1 || '%%' ESCAPE '\\'", + "x NOT LIKE '%' || :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_not_contains_autoescape(self): self.assert_compile( ~column('x').contains('a%b_c', autoescape='\\'), - "x NOT LIKE '%%' || :x_1 || '%%' ESCAPE '\\'", + "x NOT LIKE '%' || :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) @@ -2402,56 +2402,56 @@ class ComposedLikeOperatorsTest(fixtures.TestBase, testing.AssertsCompiledSQL): def test_startswith(self): self.assert_compile( column('x').startswith('y'), - "x LIKE :x_1 || '%%'", + "x LIKE :x_1 || '%'", checkparams={'x_1': 'y'} ) def test_startswith_escape(self): self.assert_compile( column('x').startswith('a%b_c', escape='\\'), - "x LIKE :x_1 || '%%' ESCAPE '\\'", + "x LIKE :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_startswith_autoescape(self): self.assert_compile( column('x').startswith('a%b_c', autoescape='\\'), - "x LIKE :x_1 || '%%' ESCAPE '\\'", + "x LIKE :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) def test_not_startswith(self): self.assert_compile( ~column('x').startswith('y'), - "x NOT LIKE :x_1 || '%%'", + "x NOT LIKE :x_1 || '%'", checkparams={'x_1': 'y'} ) def test_not_startswith_escape(self): self.assert_compile( ~column('x').startswith('a%b_c', escape='\\'), - "x NOT LIKE :x_1 || '%%' ESCAPE '\\'", + "x NOT LIKE :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_not_startswith_autoescape(self): self.assert_compile( ~column('x').startswith('a%b_c', autoescape='\\'), - "x NOT LIKE :x_1 || '%%' ESCAPE '\\'", + "x NOT LIKE :x_1 || '%' ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) def test_startswith_literal(self): self.assert_compile( column('x').startswith(literal_column('y')), - "x LIKE y || '%%'", + "x LIKE y || '%'", checkparams={} ) def test_startswith_text(self): self.assert_compile( column('x').startswith(text('y')), - "x LIKE y || '%%'", + "x LIKE y || '%'", checkparams={} ) @@ -2506,56 +2506,56 @@ class ComposedLikeOperatorsTest(fixtures.TestBase, testing.AssertsCompiledSQL): def test_endswith(self): self.assert_compile( column('x').endswith('y'), - "x LIKE '%%' || :x_1", + "x LIKE '%' || :x_1", checkparams={'x_1': 'y'} ) def test_endswith_escape(self): self.assert_compile( column('x').endswith('a%b_c', escape='\\'), - "x LIKE '%%' || :x_1 ESCAPE '\\'", + "x LIKE '%' || :x_1 ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_endswith_autoescape(self): self.assert_compile( column('x').endswith('a%b_c', autoescape='\\'), - "x LIKE '%%' || :x_1 ESCAPE '\\'", + "x LIKE '%' || :x_1 ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) def test_not_endswith(self): self.assert_compile( ~column('x').endswith('y'), - "x NOT LIKE '%%' || :x_1", + "x NOT LIKE '%' || :x_1", checkparams={'x_1': 'y'} ) def test_not_endswith_escape(self): self.assert_compile( ~column('x').endswith('a%b_c', escape='\\'), - "x NOT LIKE '%%' || :x_1 ESCAPE '\\'", + "x NOT LIKE '%' || :x_1 ESCAPE '\\'", checkparams={'x_1': 'a%b_c'} ) def test_not_endswith_autoescape(self): self.assert_compile( ~column('x').endswith('a%b_c', autoescape='\\'), - "x NOT LIKE '%%' || :x_1 ESCAPE '\\'", + "x NOT LIKE '%' || :x_1 ESCAPE '\\'", checkparams={'x_1': 'a\\%b\\_c'} ) def test_endswith_literal(self): self.assert_compile( column('x').endswith(literal_column('y')), - "x LIKE '%%' || y", + "x LIKE '%' || y", checkparams={} ) def test_endswith_text(self): self.assert_compile( column('x').endswith(text('y')), - "x LIKE '%%' || y", + "x LIKE '%' || y", checkparams={} ) diff --git a/test/sql/test_quote.py b/test/sql/test_quote.py index a436dde670..477fca7836 100644 --- a/test/sql/test_quote.py +++ b/test/sql/test_quote.py @@ -1,12 +1,12 @@ -from sqlalchemy import * +from sqlalchemy import MetaData, Table, Column, Integer, select, \ + ForeignKey, Index, CheckConstraint, inspect, column from sqlalchemy import sql, schema from sqlalchemy.sql import compiler from sqlalchemy.testing import fixtures, AssertsCompiledSQL, eq_ from sqlalchemy import testing -from sqlalchemy.sql.elements import (quoted_name, - _truncated_label, - _anonymous_label) +from sqlalchemy.sql.elements import quoted_name, _anonymous_label from sqlalchemy.testing.util import picklers +from sqlalchemy.engine import default class QuoteExecTest(fixtures.TestBase): @@ -700,7 +700,7 @@ class PreparerTest(fixtures.TestBase): """Test the db-agnostic quoting services of IdentifierPreparer.""" def test_unformat(self): - prep = compiler.IdentifierPreparer(None) + prep = compiler.IdentifierPreparer(default.DefaultDialect()) unformat = prep.unformat_identifiers def a_eq(have, want): @@ -732,7 +732,7 @@ class PreparerTest(fixtures.TestBase): def _unescape_identifier(self, value): return value.replace('``', '`') - prep = Custom(None) + prep = Custom(default.DefaultDialect()) unformat = prep.unformat_identifiers def a_eq(have, want): diff --git a/test/sql/test_text.py b/test/sql/test_text.py index 4a273e1ee2..ca4d10702a 100644 --- a/test/sql/test_text.py +++ b/test/sql/test_text.py @@ -317,6 +317,19 @@ class BindParamTest(fixtures.TestBase, AssertsCompiledSQL): checkparams={'y': 6, 'x': 5, 'z': 7} ) + def test_escaping_percent_signs(self): + stmt = text("select '%' where foo like '%bar%'") + self.assert_compile( + stmt, + "select '%' where foo like '%bar%'", + dialect="sqlite" + ) + + self.assert_compile( + stmt, + "select '%%' where foo like '%%bar%%'", + dialect="mysql" + ) class AsFromTest(fixtures.TestBase, AssertsCompiledSQL): __dialect__ = 'default'