--- /dev/null
+.. change::
+ :tags: bug, sql
+ :tickets: 10042
+ :versions: 2.0.18
+
+ Fixed issue where the :meth:`_sql.ColumnOperators.regexp_match`
+ when using "flags" would not produce a "stable" cache key, that
+ is, the cache key would keep changing each time causing cache pollution.
+ The same issue existed for :meth:`_sql.ColumnOperators.regexp_replace`
+ with both the flags and the actual replacement expression.
+ The flags are now represented as fixed modifier strings rendered as
+ safestrings rather than bound parameters, and the replacement
+ expression is established within the primary portion of the "binary"
+ element so that it generates an appropriate cache key.
+
+ Note that as part of this change, the
+ :paramref:`_sql.ColumnOperators.regexp_match.flags` and
+ :paramref:`_sql.ColumnOperators.regexp_replace.flags` have been modified to
+ render as literal strings only, whereas previously they were rendered as
+ full SQL expressions, typically bound parameters. These parameters should
+ always be passed as plain Python strings and not as SQL expression
+ constructs; it's not expected that SQL expression constructs were used in
+ practice for this parameter, so this is a backwards-incompatible change.
+
+ The change also modifies the internal structure of the expression
+ generated, for :meth:`_sql.ColumnOperators.regexp_replace` with or without
+ flags, and for :meth:`_sql.ColumnOperators.regexp_match` with flags. Third
+ party dialects which may have implemented regexp implementations of their
+ own (no such dialects could be located in a search, so impact is expected
+ to be low) would need to adjust the traversal of the structure to
+ accommodate.
+
+
+.. change::
+ :tags: bug, sql
+ :versions: 2.0.18
+
+ Fixed issue in mostly-internal :class:`.CacheKey` construct where the
+ ``__ne__()`` operator were not properly implemented, leading to nonsensical
+ results when comparing :class:`.CacheKey` instances to each other.
+
+
+
def _mariadb_regexp_flags(self, flags, pattern, **kw):
return "CONCAT('(?', %s, ')', %s)" % (
- self.process(flags, **kw),
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
self.process(pattern, **kw),
)
text = "REGEXP_LIKE(%s, %s, %s)" % (
self.process(binary.left, **kw),
self.process(binary.right, **kw),
- self.process(flags, **kw),
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
)
if op_string == " NOT REGEXP ":
return "NOT %s" % text
def visit_regexp_replace_op_binary(self, binary, operator, **kw):
flags = binary.modifiers["flags"]
- replacement = binary.modifiers["replacement"]
if flags is None:
- return "REGEXP_REPLACE(%s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s)" % (
self.process(binary.left, **kw),
self.process(binary.right, **kw),
- self.process(replacement, **kw),
)
elif self.dialect.is_mariadb:
return "REGEXP_REPLACE(%s, %s, %s)" % (
self.process(binary.left, **kw),
- self._mariadb_regexp_flags(flags, binary.right),
- self.process(replacement, **kw),
+ self._mariadb_regexp_flags(flags, binary.right.clauses[0]),
+ self.process(binary.right.clauses[1], **kw),
)
else:
- return "REGEXP_REPLACE(%s, %s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s, %s)" % (
self.process(binary.left, **kw),
self.process(binary.right, **kw),
- self.process(replacement, **kw),
- self.process(flags, **kw),
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
)
return "REGEXP_LIKE(%s, %s, %s)" % (
string,
pattern,
- self.process(flags, **kw),
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
)
def visit_not_regexp_match_op_binary(self, binary, operator, **kw):
def visit_regexp_replace_op_binary(self, binary, operator, **kw):
string = self.process(binary.left, **kw)
- pattern = self.process(binary.right, **kw)
- replacement = self.process(binary.modifiers["replacement"], **kw)
+ pattern_replace = self.process(binary.right, **kw)
flags = binary.modifiers["flags"]
if flags is None:
- return "REGEXP_REPLACE(%s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s)" % (
string,
- pattern,
- replacement,
+ pattern_replace,
)
else:
- return "REGEXP_REPLACE(%s, %s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s, %s)" % (
string,
- pattern,
- replacement,
- self.process(flags, **kw),
+ pattern_replace,
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
)
return self._generate_generic_binary(
binary, " %s " % base_op, **kw
)
- if isinstance(flags, elements.BindParameter) and flags.value == "i":
+ if flags == "i":
return self._generate_generic_binary(
binary, " %s* " % base_op, **kw
)
return "%s %s CONCAT('(?', %s, ')', %s)" % (
self.process(binary.left, **kw),
base_op,
- self.process(flags, **kw),
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
self.process(binary.right, **kw),
)
def visit_regexp_replace_op_binary(self, binary, operator, **kw):
string = self.process(binary.left, **kw)
- pattern = self.process(binary.right, **kw)
+ pattern_replace = self.process(binary.right, **kw)
flags = binary.modifiers["flags"]
- replacement = self.process(binary.modifiers["replacement"], **kw)
if flags is None:
- return "REGEXP_REPLACE(%s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s)" % (
string,
- pattern,
- replacement,
+ pattern_replace,
)
else:
- return "REGEXP_REPLACE(%s, %s, %s, %s)" % (
+ return "REGEXP_REPLACE(%s, %s, %s)" % (
string,
- pattern,
- replacement,
- self.process(flags, **kw),
+ pattern_replace,
+ self.render_literal_value(flags, sqltypes.STRINGTYPE),
)
def visit_empty_set_expr(self, element_types, **kw):
def __eq__(self, other: Any) -> bool:
return bool(self.key == other.key)
+ def __ne__(self, other: Any) -> bool:
+ return not (self.key == other.key)
+
@classmethod
def _diff_tuples(cls, left: CacheKey, right: CacheKey) -> str:
ck1 = CacheKey(left, [])
return self._generate_generic_binary(binary, " <not regexp> ", **kw)
def visit_regexp_replace_op_binary(self, binary, operator, **kw):
- replacement = binary.modifiers["replacement"]
- return "<regexp replace>(%s, %s, %s)" % (
+ return "<regexp replace>(%s, %s)" % (
binary.left._compiler_dispatch(self, **kw),
binary.right._compiler_dispatch(self, **kw),
- replacement._compiler_dispatch(self, **kw),
)
def visit_try_cast(self, cast, **kwargs):
flags: Optional[str],
**kw: Any,
) -> ColumnElement[Any]:
- if flags is not None:
- flags_expr = coercions.expect(
+ return BinaryExpression(
+ expr,
+ coercions.expect(
roles.BinaryElementRole,
- flags,
+ pattern,
expr=expr,
- operator=operators.regexp_replace_op,
- )
- else:
- flags_expr = None
- return _boolean_compare(
- expr,
+ operator=operators.comma_op,
+ ),
op,
- pattern,
- flags=flags_expr,
- negate_op=operators.not_regexp_match_op
- if op is operators.regexp_match_op
- else operators.regexp_match_op,
- **kw,
+ negate=operators.not_regexp_match_op,
+ modifiers={"flags": flags},
)
flags: Optional[str],
**kw: Any,
) -> ColumnElement[Any]:
- replacement = coercions.expect(
- roles.BinaryElementRole,
- replacement,
- expr=expr,
- operator=operators.regexp_replace_op,
- )
- if flags is not None:
- flags_expr = coercions.expect(
- roles.BinaryElementRole,
- flags,
- expr=expr,
- operator=operators.regexp_replace_op,
- )
- else:
- flags_expr = None
- return _binary_operate(
- expr, op, pattern, replacement=replacement, flags=flags_expr, **kw
+ return BinaryExpression(
+ expr,
+ ExpressionClauseList._construct_for_list(
+ operators.comma_op,
+ type_api.NULLTYPE,
+ coercions.expect(
+ roles.BinaryElementRole,
+ pattern,
+ expr=expr,
+ operator=operators.comma_op,
+ ),
+ coercions.expect(
+ roles.BinaryElementRole,
+ replacement,
+ expr=expr,
+ operator=operators.comma_op,
+ ),
+ group=False,
+ ),
+ op,
+ modifiers={"flags": flags},
)
:param pattern: The regular expression pattern string or column
clause.
- :param flags: Any regular expression string flags to apply. Flags
- tend to be backend specific. It can be a string or a column clause.
+ :param flags: Any regular expression string flags to apply, passed as
+ plain Python string only. These flags are backend specific.
Some backends, like PostgreSQL and MariaDB, may alternatively
specify the flags as part of the pattern.
When using the ignore case flag 'i' in PostgreSQL, the ignore case
.. versionadded:: 1.4
+ .. versionchanged:: 1.4.48, 2.0.18 Note that due to an implementation
+ error, the "flags" parameter previously accepted SQL expression
+ objects such as column expressions in addition to plain Python
+ strings. This implementation did not work correctly with caching
+ and was removed; strings only should be passed for the "flags"
+ parameter, as these flags are rendered as literal inline values
+ within SQL expressions.
+
.. seealso::
:meth:`_sql.ColumnOperators.regexp_replace`
:param pattern: The regular expression pattern string or column
clause.
:param pattern: The replacement string or column clause.
- :param flags: Any regular expression string flags to apply. Flags
- tend to be backend specific. It can be a string or a column clause.
+ :param flags: Any regular expression string flags to apply, passed as
+ plain Python string only. These flags are backend specific.
Some backends, like PostgreSQL and MariaDB, may alternatively
specify the flags as part of the pattern.
.. versionadded:: 1.4
+ .. versionchanged:: 1.4.48, 2.0.18 Note that due to an implementation
+ error, the "flags" parameter previously accepted SQL expression
+ objects such as column expressions in addition to plain Python
+ strings. This implementation did not work correctly with caching
+ and was removed; strings only should be passed for the "flags"
+ parameter, as these flags are rendered as literal inline values
+ within SQL expressions.
+
+
.. seealso::
:meth:`_sql.ColumnOperators.regexp_match`
class RegexpTestMySql(fixtures.TestBase, RegexpCommon):
__dialect__ = "mysql"
+ def test_regexp_match_flags_safestring(self):
+ self.assert_compile(
+ self.table.c.myid.regexp_match("pattern", flags="i'g"),
+ "REGEXP_LIKE(mytable.myid, %s, 'i''g')",
+ checkpositional=("pattern",),
+ )
+
def test_regexp_match_flags(self):
self.assert_compile(
self.table.c.myid.regexp_match("pattern", flags="ig"),
- "REGEXP_LIKE(mytable.myid, %s, %s)",
- checkpositional=("pattern", "ig"),
+ "REGEXP_LIKE(mytable.myid, %s, 'ig')",
+ checkpositional=("pattern",),
)
def test_not_regexp_match_flags(self):
self.assert_compile(
~self.table.c.myid.regexp_match("pattern", flags="ig"),
- "NOT REGEXP_LIKE(mytable.myid, %s, %s)",
- checkpositional=("pattern", "ig"),
+ "NOT REGEXP_LIKE(mytable.myid, %s, 'ig')",
+ checkpositional=("pattern",),
)
def test_regexp_replace_flags(self):
self.table.c.myid.regexp_replace(
"pattern", "replacement", flags="ig"
),
- "REGEXP_REPLACE(mytable.myid, %s, %s, %s)",
- checkpositional=("pattern", "replacement", "ig"),
+ "REGEXP_REPLACE(mytable.myid, %s, %s, 'ig')",
+ checkpositional=("pattern", "replacement"),
+ )
+
+ def test_regexp_replace_flags_safestring(self):
+ self.assert_compile(
+ self.table.c.myid.regexp_replace(
+ "pattern", "replacement", flags="i'g"
+ ),
+ "REGEXP_REPLACE(mytable.myid, %s, %s, 'i''g')",
+ checkpositional=("pattern", "replacement"),
)
class RegexpTestMariaDb(fixtures.TestBase, RegexpCommon):
__dialect__ = "mariadb"
+ def test_regexp_match_flags_safestring(self):
+ self.assert_compile(
+ self.table.c.myid.regexp_match("pattern", flags="i'g"),
+ "mytable.myid REGEXP CONCAT('(?', 'i''g', ')', %s)",
+ checkpositional=("pattern",),
+ )
+
def test_regexp_match_flags(self):
self.assert_compile(
self.table.c.myid.regexp_match("pattern", flags="ig"),
- "mytable.myid REGEXP CONCAT('(?', %s, ')', %s)",
- checkpositional=("ig", "pattern"),
+ "mytable.myid REGEXP CONCAT('(?', 'ig', ')', %s)",
+ checkpositional=("pattern",),
)
def test_not_regexp_match_flags(self):
self.assert_compile(
~self.table.c.myid.regexp_match("pattern", flags="ig"),
- "mytable.myid NOT REGEXP CONCAT('(?', %s, ')', %s)",
- checkpositional=("ig", "pattern"),
+ "mytable.myid NOT REGEXP CONCAT('(?', 'ig', ')', %s)",
+ checkpositional=("pattern",),
)
def test_regexp_replace_flags(self):
self.table.c.myid.regexp_replace(
"pattern", "replacement", flags="ig"
),
- "REGEXP_REPLACE(mytable.myid, CONCAT('(?', %s, ')', %s), %s)",
- checkpositional=("ig", "pattern", "replacement"),
+ "REGEXP_REPLACE(mytable.myid, CONCAT('(?', 'ig', ')', %s), %s)",
+ checkpositional=("pattern", "replacement"),
)
def test_regexp_match_flags(self):
self.assert_compile(
self.table.c.myid.regexp_match("pattern", flags="ig"),
- "REGEXP_LIKE(mytable.myid, :myid_1, :myid_2)",
- checkparams={"myid_1": "pattern", "myid_2": "ig"},
+ "REGEXP_LIKE(mytable.myid, :myid_1, 'ig')",
+ checkparams={"myid_1": "pattern"},
)
- def test_regexp_match_flags_col(self):
+ def test_regexp_match_flags_safestring(self):
self.assert_compile(
- self.table.c.myid.regexp_match("pattern", flags=self.table.c.name),
- "REGEXP_LIKE(mytable.myid, :myid_1, mytable.name)",
+ self.table.c.myid.regexp_match("pattern", flags="i'g"),
+ "REGEXP_LIKE(mytable.myid, :myid_1, 'i''g')",
checkparams={"myid_1": "pattern"},
)
checkparams={"param_1": "string"},
)
- def test_not_regexp_match_flags_col(self):
- self.assert_compile(
- ~self.table.c.myid.regexp_match(
- "pattern", flags=self.table.c.name
- ),
- "NOT REGEXP_LIKE(mytable.myid, :myid_1, mytable.name)",
- checkparams={"myid_1": "pattern"},
- )
-
def test_not_regexp_match_flags(self):
self.assert_compile(
~self.table.c.myid.regexp_match("pattern", flags="ig"),
- "NOT REGEXP_LIKE(mytable.myid, :myid_1, :myid_2)",
- checkparams={"myid_1": "pattern", "myid_2": "ig"},
+ "NOT REGEXP_LIKE(mytable.myid, :myid_1, 'ig')",
+ checkparams={"myid_1": "pattern"},
)
def test_regexp_replace(self):
self.table.c.myid.regexp_replace(
"pattern", "replacement", flags="ig"
),
- "REGEXP_REPLACE(mytable.myid, :myid_1, :myid_2, :myid_3)",
+ "REGEXP_REPLACE(mytable.myid, :myid_1, :myid_2, 'ig')",
checkparams={
"myid_1": "pattern",
"myid_2": "replacement",
- "myid_3": "ig",
},
)
- def test_regexp_replace_flags_col(self):
+ def test_regexp_replace_flags_safestring(self):
self.assert_compile(
self.table.c.myid.regexp_replace(
- "pattern", "replacement", flags=self.table.c.name
+ "pattern", "replacement", flags="i'g"
),
- "REGEXP_REPLACE(mytable.myid, :myid_1, :myid_2, mytable.name)",
- checkparams={"myid_1": "pattern", "myid_2": "replacement"},
+ "REGEXP_REPLACE(mytable.myid, :myid_1, :myid_2, 'i''g')",
+ checkparams={
+ "myid_1": "pattern",
+ "myid_2": "replacement",
+ },
)
def test_regexp_match_flags(self):
self.assert_compile(
self.table.c.myid.regexp_match("pattern", flags="ig"),
- "mytable.myid ~ CONCAT('(?', %(myid_1)s, ')', %(myid_2)s)",
- checkparams={"myid_2": "pattern", "myid_1": "ig"},
+ "mytable.myid ~ CONCAT('(?', 'ig', ')', %(myid_1)s)",
+ checkparams={"myid_1": "pattern"},
)
def test_regexp_match_flags_ignorecase(self):
checkparams={"myid_1": "pattern"},
)
- def test_regexp_match_flags_col(self):
- self.assert_compile(
- self.table.c.myid.regexp_match("pattern", flags=self.table.c.name),
- "mytable.myid ~ CONCAT('(?', mytable.name, ')', %(myid_1)s)",
- checkparams={"myid_1": "pattern"},
- )
-
def test_not_regexp_match(self):
self.assert_compile(
~self.table.c.myid.regexp_match("pattern"),
def test_not_regexp_match_flags(self):
self.assert_compile(
~self.table.c.myid.regexp_match("pattern", flags="ig"),
- "mytable.myid !~ CONCAT('(?', %(myid_1)s, ')', %(myid_2)s)",
- checkparams={"myid_2": "pattern", "myid_1": "ig"},
+ "mytable.myid !~ CONCAT('(?', 'ig', ')', %(myid_1)s)",
+ checkparams={"myid_1": "pattern"},
)
def test_not_regexp_match_flags_ignorecase(self):
checkparams={"myid_1": "pattern"},
)
- def test_not_regexp_match_flags_col(self):
- self.assert_compile(
- ~self.table.c.myid.regexp_match(
- "pattern", flags=self.table.c.name
- ),
- "mytable.myid !~ CONCAT('(?', mytable.name, ')', %(myid_1)s)",
- checkparams={"myid_1": "pattern"},
- )
-
def test_regexp_replace(self):
self.assert_compile(
self.table.c.myid.regexp_replace("pattern", "replacement"),
self.table.c.myid.regexp_replace(
"pattern", "replacement", flags="ig"
),
- "REGEXP_REPLACE(mytable.myid, %(myid_1)s, %(myid_2)s, %(myid_3)s)",
+ "REGEXP_REPLACE(mytable.myid, %(myid_1)s, %(myid_2)s, 'ig')",
checkparams={
"myid_1": "pattern",
"myid_2": "replacement",
- "myid_3": "ig",
},
)
- def test_regexp_replace_flags_col(self):
+ def test_regexp_replace_flags_safestring(self):
self.assert_compile(
self.table.c.myid.regexp_replace(
- "pattern", "replacement", flags=self.table.c.name
+ "pattern", "replacement", flags="i'g"
),
- "REGEXP_REPLACE(mytable.myid, %(myid_1)s,"
- " %(myid_2)s, mytable.name)",
- checkparams={"myid_1": "pattern", "myid_2": "replacement"},
+ "REGEXP_REPLACE(mytable.myid, %(myid_1)s, %(myid_2)s, 'i''g')",
+ checkparams={
+ "myid_1": "pattern",
+ "myid_2": "replacement",
+ },
)
@testing.combinations(
column("q").like("somstr", escape="\\"),
column("q").like("somstr", escape="X"),
),
+ lambda: (
+ column("q").regexp_match("y", flags="ig"),
+ column("q").regexp_match("y", flags="q"),
+ column("q").regexp_match("y"),
+ column("q").regexp_replace("y", "z", flags="ig"),
+ column("q").regexp_replace("y", "z", flags="q"),
+ column("q").regexp_replace("y", "z"),
+ ),
lambda: (
column("q", ARRAY(Integer))[3] == 5,
column("q", ARRAY(Integer))[3:5] == 5,
is_(large_v1._generate_cache_key(), None)
+ @testing.combinations(
+ (lambda: column("x"), lambda: column("x"), lambda: column("y")),
+ (
+ lambda: func.foo_bar(1, 2, 3),
+ lambda: func.foo_bar(4, 5, 6),
+ lambda: func.foo_bar_bat(1, 2, 3),
+ ),
+ )
+ def test_cache_key_object_comparators(self, lc1, lc2, lc3):
+ """test ne issue detected as part of #10042"""
+ c1 = lc1()
+ c2 = lc2()
+ c3 = lc3()
+
+ eq_(c1._generate_cache_key(), c2._generate_cache_key())
+ ne_(c1._generate_cache_key(), c3._generate_cache_key())
+ is_true(c1._generate_cache_key() == c2._generate_cache_key())
+ is_false(c1._generate_cache_key() != c2._generate_cache_key())
+ is_true(c1._generate_cache_key() != c3._generate_cache_key())
+ is_false(c1._generate_cache_key() == c3._generate_cache_key())
+
def test_cache_key(self):
for fixtures_, compare_values in [
(self.fixtures, True),
"mytable.myid LIKE :myid_1 ESCAPE '\\'",
)
+ def test_like_quote_escape(self):
+ self.assert_compile(
+ self.table1.c.myid.like("somstr", escape="'"),
+ "mytable.myid LIKE :myid_1 ESCAPE ''''",
+ )
+
def test_like_4(self):
self.assert_compile(
~self.table1.c.myid.like("somstr", escape="\\"),
projects / thirdparty / sqlalchemy / sqlalchemy.git / commitdiff
