From: Mike Bayer Date: Mon, 9 Dec 2019 23:05:00 +0000 (-0500) Subject: Use expanding IN for all literal value IN expressions X-Git-Tag: rel_1_4_0b1~585 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=60e7034a7423955cd89d5624f8769d3804ca6d82;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Use expanding IN for all literal value IN expressions The "expanding IN" feature, which generates IN expressions at query execution time which are based on the particular parameters associated with the statement execution, is now used for all IN expressions made against lists of literal values. This allows IN expressions to be fully cacheable independently of the list of values being passed, and also includes support for empty lists. For any scenario where the IN expression contains non-literal SQL expressions, the old behavior of pre-rendering for each position in the IN is maintained. The change also completes support for expanding IN with tuples, where previously type-specific bind processors weren't taking effect. As part of this change, a more explicit separation between "literal execute" and "post compile" bound parameters is being made; as the "ansi bind rules" feature is rendering bound parameters inline, as we now support "postcompile" generically, these should be used here, however we have to render literal values at execution time even for "expanding" parameters. new test fixtures etc. are added to assert everything goes to the right place. Fixes: #4645 Change-Id: Iaa2b7bfbfaaf5b80799ee17c9b8507293cba6ed1 --- diff --git a/doc/build/changelog/migration_14.rst b/doc/build/changelog/migration_14.rst index 6c91c40f4f..a0a1f1f742 100644 --- a/doc/build/changelog/migration_14.rst +++ b/doc/build/changelog/migration_14.rst @@ -212,6 +212,113 @@ refined so that it is more compatible with Core. :ticket:`4617` +.. _change_4645: + +All IN expressions render parameters for each value in the list on the fly (e.g. expanding parameters) +------------------------------------------------------------------------------------------------------ + +The "expanding IN" feature, first introduced in :ref:`change_3953`, has matured +enough such that it is clearly superior to the previous method of rendering IN +expressions. As the approach was improved to handle empty lists of values, it +is now the only means that Core / ORM will use to render lists of IN +parameters. + +The previous approach which has been present in SQLAlchemy since its first +release was that when a list of values were passed to the +:meth:`.ColumnOperators.in_` method, the list would be expanded into a series +of individual :class:`.BindParameter` objects at statement construction time. +This suffered from the limitation that it was not possible to vary the +parameter list at statement execution time based on the parameter dictionary, +which meant that string SQL statements could not be cached independently of +their parameters, nor could the parameter dictionary be fully used for +statements that included IN expressions generally. + +In order to service the "baked query" feature described at +:ref:`baked_toplevel`, a cacheable version of IN was needed, which is what +brought about the "expanding IN" feature. In contrast to the existing behavior +whereby the parameter list is expanded at statement construction time into +individual :class:`.BindParameter` objects, the feature instead uses a single +:class:`.BindParameter` that stores the list of values at once; when the +statement is executed by the :class:`.Engine`, it is "expanded" on the fly into +individual bound parameter positions based on the parameters passed to the call +to :meth:`.Connection.execute`, and the existing SQL string which may have been +retrieved from a previous execution is modified using a regular expression to +suit the current parameter set. This allows for the same :class:`.Compiled` +object, which stores the rendered string statement, to be invoked multiple +times against different parameter sets that modify the list contents passed to +IN expressions, while still maintaining the behavior of individual scalar +parameters being passed to the DBAPI. While some DBAPIs do support this +functionality directly, it is not generally available; the "expanding IN" +feature now supports the behavior consistently for all backends. + +As a major focus of 1.4 is to allow for true statement caching in Core and ORM +without the awkwardness of the "baked" system, and since the "expanding IN" +feature represents a simpler approach to building expressions in any case, +it's now invoked automatically whenever a list of values is passed to +an IN expression:: + + stmt = select([A.id, A.data]).where(A.id.in_([1, 2, 3])) + +The pre-execution string representation is:: + + >>> print(stmt) + SELECT a.id, a.data + FROM a + WHERE a.id IN ([POSTCOMPILE_id_1]) + +To render the values directly, use ``literal_binds`` as was the case previously:: + + >>> print(stmt.compile(compile_kwargs={"literal_binds": True})) + SELECT a.id, a.data + FROM a + WHERE a.id IN (1, 2, 3) + +A new flag, "render_postcompile", is added as a helper to allow the current +bound value to be rendered as it would be passed to the database:: + + >>> print(stmt.compile(compile_kwargs={"render_postcompile": True})) + SELECT a.id, a.data + FROM a + WHERE a.id IN (:id_1_1, :id_1_2, :id_1_3) + +Engine logging output shows the ultimate rendered statement as well:: + + INFO sqlalchemy.engine.base.Engine SELECT a.id, a.data + FROM a + WHERE a.id IN (?, ?, ?) + INFO sqlalchemy.engine.base.Engine (1, 2, 3) + +As part of this change, the behavior of "empty IN" expressions, where the list +parameter is empty, is now standardized on use of the IN operator against a +so-called "empty set". As there is no standard SQL syntax for empty sets, a +SELECT that returns no rows is used, tailored in specific ways for each backend +so that the database treats it as an empty set; this feature was first +introduced in version 1.3 and is described at :ref:`change_4271`. The +:paramref:`.create_engine.empty_in_strategy` parameter, introduced in version +1.2 as a means for migrating for how this case was treated for the previous IN +system, is now deprecated and this flag no longer has an effect; as described +in :ref:`change_3907`, this flag allowed a dialect to switch between the +original system of comparing a column against itself, which turned out to be a +huge performance issue, and a newer system of comparing "1 != 1" in +order to produce a "false" expression. The 1.3 introduced behavior which +now takes place in all cases is more correct than both approaches as the IN +operator is still used, and does not have the performance issue of the original +system. + +In addition, the "expanding" parameter system has been generalized so that it +also services other dialect-specific use cases where a parameter cannot be +accommodated by the DBAPI or backing database; see :ref:`change_4808` for +details. + +.. seealso:: + + :ref:`change_4808` + + :ref:`change_4271` + + :class:`.BindParameter` + +:ticket:`4645` New Features - ORM ================== diff --git a/doc/build/changelog/unreleased_14/4645.rst b/doc/build/changelog/unreleased_14/4645.rst new file mode 100644 index 0000000000..17348a65b7 --- /dev/null +++ b/doc/build/changelog/unreleased_14/4645.rst @@ -0,0 +1,18 @@ +.. change:: + :tags: feature, sql + :tickets: 4645 + + The "expanding IN" feature, which generates IN expressions at query + execution time which are based on the particular parameters associated with + the statement execution, is now used for all IN expressions made against + lists of literal values. This allows IN expressions to be fully cacheable + independently of the list of values being passed, and also includes support + for empty lists. For any scenario where the IN expression contains + non-literal SQL expressions, the old behavior of pre-rendering for each + position in the IN is maintained. The change also completes support for + expanding IN with tuples, where previously type-specific bind processors + weren't taking effect. + + .. seealso:: + + :ref:`change_4645` \ No newline at end of file diff --git a/examples/sharding/attribute_shard.py b/examples/sharding/attribute_shard.py index 608f0f9c3a..7b8f87d90b 100644 --- a/examples/sharding/attribute_shard.py +++ b/examples/sharding/attribute_shard.py @@ -17,11 +17,8 @@ from sqlalchemy.sql import operators from sqlalchemy.sql import visitors -# db1 is used for id generation. The "pool_threadlocal" -# causes the id_generator() to use the same connection as that -# of an ongoing transaction within db1. echo = True -db1 = create_engine("sqlite://", echo=echo, pool_threadlocal=True) +db1 = create_engine("sqlite://", echo=echo) db2 = create_engine("sqlite://", echo=echo) db3 = create_engine("sqlite://", echo=echo) db4 = create_engine("sqlite://", echo=echo) @@ -220,20 +217,7 @@ def _get_query_comparisons(query): clauses.add(column) def visit_binary(binary): - # special handling for "col IN (params)" - if ( - binary.left in clauses - and binary.operator == operators.in_op - and hasattr(binary.right, "clauses") - ): - comparisons.append( - ( - binary.left, - binary.operator, - tuple(binds[bind] for bind in binary.right.clauses), - ) - ) - elif binary.left in clauses and binary.right in binds: + if binary.left in clauses and binary.right in binds: comparisons.append( (binary.left, binary.operator, binds[binary.right]) ) diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index ded90c3487..cf703a3636 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -1909,14 +1909,14 @@ class MSSQLStrictCompiler(MSSQLCompiler): ansi_bind_rules = True def visit_in_op_binary(self, binary, operator, **kw): - kw["literal_binds"] = True + kw["literal_execute"] = True return "%s IN %s" % ( self.process(binary.left, **kw), self.process(binary.right, **kw), ) def visit_notin_op_binary(self, binary, operator, **kw): - kw["literal_binds"] = True + kw["literal_execute"] = True return "%s NOT IN %s" % ( self.process(binary.left, **kw), self.process(binary.right, **kw), diff --git a/lib/sqlalchemy/engine/create.py b/lib/sqlalchemy/engine/create.py index fd6105561e..1378a67994 100644 --- a/lib/sqlalchemy/engine/create.py +++ b/lib/sqlalchemy/engine/create.py @@ -23,7 +23,16 @@ from .. import util ":func:`.create_mock_engine` going forward. For general " "customization of create_engine which may have been accomplished " "using strategies, see :class:`.CreateEnginePlugin`.", - ) + ), + empty_in_strategy=( + "1.4", + "The :paramref:`.create_engine.empty_in_strategy` keyword is " + "deprecated, and no longer has any effect. All IN expressions " + "are now rendered using " + 'the "expanding parameter" strategy which renders a set of bound' + 'expressions, or an "empty set" SELECT, at statement execution' + "time.", + ), ) def create_engine(url, **kwargs): """Create a new :class:`.Engine` instance. @@ -130,23 +139,8 @@ def create_engine(url, **kwargs): logging. - :param empty_in_strategy: The SQL compilation strategy to use when - rendering an IN or NOT IN expression for :meth:`.ColumnOperators.in_` - where the right-hand side - is an empty set. This is a string value that may be one of - ``static``, ``dynamic``, or ``dynamic_warn``. The ``static`` - strategy is the default, and an IN comparison to an empty set - will generate a simple false expression "1 != 1". The ``dynamic`` - strategy behaves like that of SQLAlchemy 1.1 and earlier, emitting - a false expression of the form "expr != expr", which has the effect - of evaluting to NULL in the case of a null expression. - ``dynamic_warn`` is the same as ``dynamic``, however also emits a - warning when an empty set is encountered; this because the "dynamic" - comparison is typically poorly performing on most databases. - - .. versionadded:: 1.2 Added the ``empty_in_strategy`` setting and - additionally defaulted the behavior for empty-set IN comparisons - to a static boolean expression. + :param empty_in_strategy: No longer used; SQLAlchemy now uses + "empty set" behavior for IN in all cases. :param encoding: Defaults to ``utf-8``. This is the string encoding used by SQLAlchemy for string encode/decode @@ -412,6 +406,8 @@ def create_engine(url, **kwargs): else: raise exc.ArgumentError("unknown strategy: %r" % strat) + kwargs.pop("empty_in_strategy", None) + # create url.URL object u = _url.make_url(url) diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index 9eacf05273..7016adcf49 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -192,7 +192,16 @@ class DefaultDialect(interfaces.Dialect): "and corresponding dialect-level parameters are deprecated, " "and will be removed in a future release. Modern DBAPIs support " "Python Unicode natively and this parameter is unnecessary.", - ) + ), + empty_in_strategy=( + "1.4", + "The :paramref:`.create_engine.empty_in_strategy` keyword is " + "deprecated, and no longer has any effect. All IN expressions " + "are now rendered using " + 'the "expanding parameter" strategy which renders a set of bound' + 'expressions, or an "empty set" SELECT, at statement execution' + "time.", + ), ) def __init__( self, @@ -203,7 +212,6 @@ class DefaultDialect(interfaces.Dialect): implicit_returning=None, case_sensitive=True, supports_native_boolean=None, - empty_in_strategy="static", max_identifier_length=None, label_length=None, **kwargs @@ -235,18 +243,6 @@ class DefaultDialect(interfaces.Dialect): self.supports_native_boolean = supports_native_boolean self.case_sensitive = case_sensitive - self.empty_in_strategy = empty_in_strategy - if empty_in_strategy == "static": - self._use_static_in = True - elif empty_in_strategy in ("dynamic", "dynamic_warn"): - self._use_static_in = False - self._warn_on_empty_in = empty_in_strategy == "dynamic_warn" - else: - raise exc.ArgumentError( - "empty_in_strategy may be 'static', " - "'dynamic', or 'dynamic_warn'" - ) - self._user_defined_max_identifier_length = max_identifier_length if self._user_defined_max_identifier_length: self.max_identifier_length = ( @@ -732,19 +728,18 @@ class DefaultExecutionContext(interfaces.ExecutionContext): compiled._loose_column_name_matching, ) - self.unicode_statement = util.text_type(compiled) - if not dialect.supports_unicode_statements: - self.statement = self.unicode_statement.encode( - self.dialect.encoding - ) - else: - self.statement = self.unicode_statement - self.isinsert = compiled.isinsert self.isupdate = compiled.isupdate self.isdelete = compiled.isdelete self.is_text = compiled.isplaintext + if self.isinsert or self.isupdate or self.isdelete: + self.is_crud = True + self._is_explicit_returning = bool(compiled.statement._returning) + self._is_implicit_returning = bool( + compiled.returning and not compiled.statement._returning + ) + if not parameters: self.compiled_parameters = [compiled.construct_params()] else: @@ -755,14 +750,11 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.executemany = len(parameters) > 1 - self.cursor = self.create_cursor() + # this must occur before create_cursor() since the statement + # has to be regexed in some cases for server side cursor + self.unicode_statement = util.text_type(compiled) - if self.isinsert or self.isupdate or self.isdelete: - self.is_crud = True - self._is_explicit_returning = bool(compiled.statement._returning) - self._is_implicit_returning = bool( - compiled.returning and not compiled.statement._returning - ) + self.cursor = self.create_cursor() if self.compiled.insert_prefetch or self.compiled.update_prefetch: if self.executemany: @@ -772,15 +764,38 @@ class DefaultExecutionContext(interfaces.ExecutionContext): processors = compiled._bind_processors - if compiled.literal_execute_params: - # copy processors for this case as they will be mutated - processors = dict(processors) - positiontup = self._literal_execute_parameters( - compiled, processors + if compiled.literal_execute_params or compiled.post_compile_params: + if self.executemany: + raise exc.InvalidRequestError( + "'literal_execute' or 'expanding' parameters can't be " + "used with executemany()" + ) + + expanded_state = compiled._process_parameters_for_postcompile( + self.compiled_parameters[0] ) + + # re-assign self.unicode_statement + self.unicode_statement = expanded_state.statement + + # used by set_input_sizes() which is needed for Oracle + self._expanded_parameters = expanded_state.parameter_expansion + + processors = dict(processors) + processors.update(expanded_state.processors) + positiontup = expanded_state.positiontup elif compiled.positional: positiontup = self.compiled.positiontup + # final self.unicode_statement is now assigned, encode if needed + # by dialect + if not dialect.supports_unicode_statements: + self.statement = self.unicode_statement.encode( + self.dialect.encoding + ) + else: + self.statement = self.unicode_statement + # Convert the dictionary of bind parameter values # into a dict or list to be sent to the DBAPI's # execute() or executemany() method. @@ -825,105 +840,6 @@ class DefaultExecutionContext(interfaces.ExecutionContext): return self - def _literal_execute_parameters(self, compiled, processors): - """handle special post compile parameters. - - These include: - - * "expanding" parameters -typically IN tuples that are rendered - on a per-parameter basis for an otherwise fixed SQL statement string. - - * literal_binds compiled with the literal_execute flag. Used for - things like SQL Server "TOP N" where the driver does not accommodate - N as a bound parameter. - - """ - if self.executemany: - raise exc.InvalidRequestError( - "'literal_execute' or 'expanding' parameters can't be " - "used with executemany()" - ) - - if compiled.positional and compiled._numeric_binds: - # I'm not familiar with any DBAPI that uses 'numeric'. - # strategy would likely be to make use of numbers greater than - # the highest number present; then for expanding parameters, - # append them to the end of the parameter list. that way - # we avoid having to renumber all the existing parameters. - raise NotImplementedError( - "'post-compile' bind parameters are not supported with " - "the 'numeric' paramstyle at this time." - ) - - self._expanded_parameters = {} - - compiled_params = self.compiled_parameters[0] - if compiled.positional: - positiontup = [] - else: - positiontup = None - - replacement_expressions = {} - to_update_sets = {} - - for name in ( - compiled.positiontup - if compiled.positional - else compiled.bind_names.values() - ): - parameter = compiled.binds[name] - if parameter in compiled.literal_execute_params: - - if not parameter.expanding: - value = compiled_params.pop(name) - replacement_expressions[ - name - ] = compiled.render_literal_bindparam( - parameter, render_literal_value=value - ) - continue - - if name in replacement_expressions: - to_update = to_update_sets[name] - else: - # we are removing the parameter from compiled_params - # because it is a list value, which is not expected by - # TypeEngine objects that would otherwise be asked to - # process it. the single name is being replaced with - # individual numbered parameters for each value in the - # param. - values = compiled_params.pop(name) - - leep = compiled._literal_execute_expanding_parameter - to_update, replacement_expr = leep(name, parameter, values) - - to_update_sets[name] = to_update - replacement_expressions[name] = replacement_expr - - if not parameter.literal_execute: - compiled_params.update(to_update) - - processors.update( - (key, processors[name]) - for key, value in to_update - if name in processors - ) - if compiled.positional: - positiontup.extend(name for name, value in to_update) - self._expanded_parameters[name] = [ - expand_key for expand_key, value in to_update - ] - elif compiled.positional: - positiontup.append(name) - - def process_expanding(m): - return replacement_expressions[m.group(1)] - - self.statement = re.sub( - r"\[POSTCOMPILE_(\S+)\]", process_expanding, self.statement - ) - return positiontup - @classmethod def _init_statement( cls, dialect, connection, dbapi_connection, statement, parameters @@ -1084,8 +1000,8 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.compiled.statement, expression.TextClause ) ) - and self.statement - and SERVER_SIDE_CURSOR_RE.match(self.statement) + and self.unicode_statement + and SERVER_SIDE_CURSOR_RE.match(self.unicode_statement) ) ) ) diff --git a/lib/sqlalchemy/sql/coercions.py b/lib/sqlalchemy/sql/coercions.py index b45ef39911..97524bc6a6 100644 --- a/lib/sqlalchemy/sql/coercions.py +++ b/lib/sqlalchemy/sql/coercions.py @@ -331,20 +331,29 @@ class InElementImpl(RoleImpl, roles.InElementRole): if isinstance(element, collections_abc.Iterable) and not isinstance( element, util.string_types ): - args = [] + non_literal_expressions = {} + element = list(element) for o in element: if not _is_literal(o): if not isinstance(o, operators.ColumnOperators): self._raise_for_expected(element, **kw) + else: + non_literal_expressions[o] = o elif o is None: - o = elements.Null() - else: - o = expr._bind_param(operator, o) - args.append(o) - - return elements.ClauseList( - _tuple_values=isinstance(expr, elements.Tuple), *args - ) + non_literal_expressions[o] = elements.Null() + + if non_literal_expressions: + return elements.ClauseList( + _tuple_values=isinstance(expr, elements.Tuple), + *[ + non_literal_expressions[o] + if o in non_literal_expressions + else expr._bind_param(operator, o) + for o in element + ] + ) + else: + return expr._bind_param(operator, element, expanding=True) else: self._raise_for_expected(element, **kw) @@ -353,17 +362,8 @@ class InElementImpl(RoleImpl, roles.InElementRole): if element._is_select_statement: return element.scalar_subquery() elif isinstance(element, elements.ClauseList): - if len(element.clauses) == 0: - op, negate_op = ( - (operators.empty_in_op, operators.empty_notin_op) - if operator is operators.in_op - else (operators.empty_notin_op, operators.empty_in_op) - ) - return element.self_group(against=op)._annotate( - dict(in_ops=(op, negate_op)) - ) - else: - return element.self_group(against=operator) + assert not len(element.clauses) == 0 + return element.self_group(against=operator) elif isinstance(element, elements.BindParameter) and element.expanding: if isinstance(expr, elements.Tuple): diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 807b01c243..75ccad3fd1 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -23,6 +23,7 @@ To generate user-defined SQL strings, see """ +import collections import contextlib import itertools import re @@ -257,6 +258,18 @@ RM_OBJECTS = 2 RM_TYPE = 3 +ExpandedState = collections.namedtuple( + "ExpandedState", + [ + "statement", + "additional_parameters", + "processors", + "positiontup", + "parameter_expansion", + ], +) + + class Compiled(object): """Represent a compiled SQL or DDL expression. @@ -525,6 +538,12 @@ class SQLCompiler(Compiled): """ + _render_postcompile = False + """ + whether to render out POSTCOMPILE params during the compile phase. + + """ + insert_single_values_expr = None """When an INSERT is compiled with a single set of parameters inside a VALUES expression, the string is assigned here, where it can be @@ -535,6 +554,16 @@ class SQLCompiler(Compiled): """ literal_execute_params = frozenset() + """bindparameter objects that are rendered as literal values at statement + execution time. + + """ + + post_compile_params = frozenset() + """bindparameter objects that are rendered as bound parameter placeholders + at statement execution time. + + """ insert_prefetch = update_prefetch = () @@ -610,6 +639,9 @@ class SQLCompiler(Compiled): if self.positional and self._numeric_binds: self._apply_numbered_params() + if self._render_postcompile: + self._process_parameters_for_postcompile(_populate_self=True) + @property def prefetch(self): return list(self.insert_prefetch + self.update_prefetch) @@ -665,7 +697,12 @@ class SQLCompiler(Compiled): for key, value in ( ( self.bind_names[bindparam], - bindparam.type._cached_bind_processor(self.dialect), + bindparam.type._cached_bind_processor(self.dialect) + if not bindparam._expanding_in_types + else tuple( + elem_type._cached_bind_processor(self.dialect) + for elem_type in bindparam._expanding_in_types + ), ) for bindparam in self.bind_names ) @@ -741,6 +778,141 @@ class SQLCompiler(Compiled): compiled object, for those values that are present.""" return self.construct_params(_check=False) + def _process_parameters_for_postcompile( + self, parameters=None, _populate_self=False + ): + """handle special post compile parameters. + + These include: + + * "expanding" parameters -typically IN tuples that are rendered + on a per-parameter basis for an otherwise fixed SQL statement string. + + * literal_binds compiled with the literal_execute flag. Used for + things like SQL Server "TOP N" where the driver does not accommodate + N as a bound parameter. + + """ + + if parameters is None: + parameters = self.construct_params() + + expanded_parameters = {} + if self.positional: + positiontup = [] + else: + positiontup = None + + processors = self._bind_processors + + new_processors = {} + + if self.positional and self._numeric_binds: + # I'm not familiar with any DBAPI that uses 'numeric'. + # strategy would likely be to make use of numbers greater than + # the highest number present; then for expanding parameters, + # append them to the end of the parameter list. that way + # we avoid having to renumber all the existing parameters. + raise NotImplementedError( + "'post-compile' bind parameters are not supported with " + "the 'numeric' paramstyle at this time." + ) + + replacement_expressions = {} + to_update_sets = {} + + for name in ( + self.positiontup if self.positional else self.bind_names.values() + ): + parameter = self.binds[name] + if parameter in self.literal_execute_params: + value = parameters.pop(name) + replacement_expressions[name] = self.render_literal_bindparam( + parameter, render_literal_value=value + ) + continue + + if parameter in self.post_compile_params: + if name in replacement_expressions: + to_update = to_update_sets[name] + else: + # we are removing the parameter from parameters + # because it is a list value, which is not expected by + # TypeEngine objects that would otherwise be asked to + # process it. the single name is being replaced with + # individual numbered parameters for each value in the + # param. + values = parameters.pop(name) + + leep = self._literal_execute_expanding_parameter + to_update, replacement_expr = leep(name, parameter, values) + + to_update_sets[name] = to_update + replacement_expressions[name] = replacement_expr + + if not parameter.literal_execute: + parameters.update(to_update) + if parameter._expanding_in_types: + new_processors.update( + ( + "%s_%s_%s" % (name, i, j), + processors[name][j - 1], + ) + for i, tuple_element in enumerate(values, 1) + for j, value in enumerate(tuple_element, 1) + if name in processors + and processors[name][j - 1] is not None + ) + else: + new_processors.update( + (key, processors[name]) + for key, value in to_update + if name in processors + ) + if self.positional: + positiontup.extend(name for name, value in to_update) + expanded_parameters[name] = [ + expand_key for expand_key, value in to_update + ] + elif self.positional: + positiontup.append(name) + + def process_expanding(m): + return replacement_expressions[m.group(1)] + + statement = re.sub( + r"\[POSTCOMPILE_(\S+)\]", process_expanding, self.string + ) + + expanded_state = ExpandedState( + statement, + parameters, + new_processors, + positiontup, + expanded_parameters, + ) + + if _populate_self: + # this is for the "render_postcompile" flag, which is not + # otherwise used internally and is for end-user debugging and + # special use cases. + self.string = expanded_state.statement + self._bind_processors.update(expanded_state.processors) + self.positiontup = expanded_state.positiontup + self.post_compile_params = frozenset() + for key in expanded_state.parameter_expansion: + bind = self.binds.pop(key) + self.bind_names.pop(bind) + for value, expanded_key in zip( + bind.value, expanded_state.parameter_expansion[key] + ): + self.binds[expanded_key] = new_param = bind._with_value( + value + ) + self.bind_names[new_param] = expanded_key + + return expanded_state + @util.dependencies("sqlalchemy.engine.result") def _create_result_map(self, result): """utility method used for unit tests only.""" @@ -1291,31 +1463,6 @@ class SQLCompiler(Compiled): binary, override_operator=operators.match_op ) - def _emit_empty_in_warning(self): - util.warn( - "The IN-predicate was invoked with an " - "empty sequence. This results in a " - "contradiction, which nonetheless can be " - "expensive to evaluate. Consider alternative " - "strategies for improved performance." - ) - - def visit_empty_in_op_binary(self, binary, operator, **kw): - if self.dialect._use_static_in: - return "1 != 1" - else: - if self.dialect._warn_on_empty_in: - self._emit_empty_in_warning() - return self.process(binary.left != binary.left) - - def visit_empty_notin_op_binary(self, binary, operator, **kw): - if self.dialect._use_static_in: - return "1 = 1" - else: - if self.dialect._warn_on_empty_in: - self._emit_empty_in_warning() - return self.process(binary.left == binary.left) - def visit_empty_set_expr(self, element_types): raise NotImplementedError( "Dialect '%s' does not support empty set expression." @@ -1407,7 +1554,7 @@ class SQLCompiler(Compiled): and isinstance(binary.left, elements.BindParameter) and isinstance(binary.right, elements.BindParameter) ): - kw["literal_binds"] = True + kw["literal_execute"] = True operator_ = override_operator or binary.operator disp = self._get_operator_dispatch(operator_, "binary", None) @@ -1588,6 +1735,7 @@ class SQLCompiler(Compiled): literal_binds=False, skip_bind_expression=False, literal_execute=False, + render_postcompile=False, **kwargs ): @@ -1605,17 +1753,16 @@ class SQLCompiler(Compiled): ) if not literal_binds: - post_compile = ( + literal_execute = ( literal_execute or bindparam.literal_execute - or bindparam.expanding + or (within_columns_clause and self.ansi_bind_rules) ) + post_compile = literal_execute or bindparam.expanding else: post_compile = False - if not literal_execute and ( - literal_binds or (within_columns_clause and self.ansi_bind_rules) - ): + if not literal_execute and (literal_binds): ret = self.render_literal_bindparam( bindparam, within_columns_clause=True, **kwargs ) @@ -1650,7 +1797,13 @@ class SQLCompiler(Compiled): self.binds[bindparam.key] = self.binds[name] = bindparam if post_compile: - self.literal_execute_params |= {bindparam} + if render_postcompile: + self._render_postcompile = True + + if literal_execute: + self.literal_execute_params |= {bindparam} + else: + self.post_compile_params |= {bindparam} ret = self.bindparam_string( name, @@ -2897,6 +3050,9 @@ class StrSQLCompiler(SQLCompiler): for t in extra_froms ) + def visit_empty_set_expr(self, type_): + return "SELECT 1 WHERE 1!=1" + class DDLCompiler(Compiled): @util.memoized_property diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index 464c2a4d6f..7d857d4feb 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -759,7 +759,7 @@ class ColumnElement( def reverse_operate(self, op, other, **kwargs): return op(other, self.comparator, **kwargs) - def _bind_param(self, operator, obj, type_=None): + def _bind_param(self, operator, obj, type_=None, expanding=False): return BindParameter( None, obj, @@ -767,6 +767,7 @@ class ColumnElement( type_=type_, _compared_to_type=self.type, unique=True, + expanding=expanding, ) @property @@ -1281,7 +1282,6 @@ class BindParameter(roles.InElementRole, ColumnElement): self.required = required self.expanding = expanding self.literal_execute = literal_execute - if type_ is None: if _compared_to_type is not None: self.type = _compared_to_type.coerce_compared_value( @@ -2282,20 +2282,29 @@ class Tuple(ClauseList, ColumnElement): def _select_iterable(self): return (self,) - def _bind_param(self, operator, obj, type_=None): - return Tuple( - *[ - BindParameter( - None, - o, - _compared_to_operator=operator, - _compared_to_type=compared_to_type, - unique=True, - type_=type_, - ) - for o, compared_to_type in zip(obj, self._type_tuple) - ] - ).self_group() + def _bind_param(self, operator, obj, type_=None, expanding=False): + if expanding: + return BindParameter( + None, + value=obj, + _compared_to_operator=operator, + unique=True, + expanding=True, + )._with_expanding_in_types(self._type_tuple) + else: + return Tuple( + *[ + BindParameter( + None, + o, + _compared_to_operator=operator, + _compared_to_type=compared_to_type, + unique=True, + type_=type_, + ) + for o, compared_to_type in zip(obj, self._type_tuple) + ] + ).self_group() class Case(ColumnElement): @@ -4240,7 +4249,7 @@ class ColumnClause( else: return name - def _bind_param(self, operator, obj, type_=None): + def _bind_param(self, operator, obj, type_=None, expanding=False): return BindParameter( self.key, obj, @@ -4248,6 +4257,7 @@ class ColumnClause( _compared_to_type=self.type, type_=type_, unique=True, + expanding=expanding, ) def _make_proxy( diff --git a/lib/sqlalchemy/sql/operators.py b/lib/sqlalchemy/sql/operators.py index 3aeaaa601f..22bf3d1503 100644 --- a/lib/sqlalchemy/sql/operators.py +++ b/lib/sqlalchemy/sql/operators.py @@ -538,17 +538,15 @@ class ColumnOperators(Operators): stmt.where(column.in_([])) - In this calling form, the expression renders a "false" expression, - e.g.:: + In this calling form, the expression renders an "empty set" + expression. These expressions are tailored to individual backends + and are generaly trying to get an empty SELECT statement as a + subuqery. Such as on SQLite, the expression is:: - WHERE 1 != 1 + WHERE col IN (SELECT 1 FROM (SELECT 1) WHERE 1!=1) - This "false" expression has historically had different behaviors - in older SQLAlchemy versions, see - :paramref:`.create_engine.empty_in_strategy` for behavioral options. - - .. versionchanged:: 1.2 simplified the behavior of "empty in" - expressions + .. versionchanged:: 1.4 empty IN expressions now use an + execution-time generated SELECT subquery in all cases. * A bound parameter, e.g. :func:`.bindparam`, may be used if it includes the :paramref:`.bindparam.expanding` flag:: @@ -1341,16 +1339,6 @@ def comma_op(a, b): raise NotImplementedError() -@comparison_op -def empty_in_op(a, b): - raise NotImplementedError() - - -@comparison_op -def empty_notin_op(a, b): - raise NotImplementedError() - - def filter_op(a, b): raise NotImplementedError() @@ -1473,8 +1461,6 @@ _PRECEDENCE = { ne: 5, is_distinct_from: 5, isnot_distinct_from: 5, - empty_in_op: 5, - empty_notin_op: 5, gt: 5, lt: 5, ge: 5, diff --git a/lib/sqlalchemy/testing/assertions.py b/lib/sqlalchemy/testing/assertions.py index f057ae37b2..563dc2a24f 100644 --- a/lib/sqlalchemy/testing/assertions.py +++ b/lib/sqlalchemy/testing/assertions.py @@ -340,12 +340,15 @@ class AssertsCompiledSQL(object): result, params=None, checkparams=None, + check_literal_execute=None, + check_post_param=None, dialect=None, checkpositional=None, check_prefetch=None, use_default_dialect=False, allow_dialect_select=False, literal_binds=False, + render_postcompile=False, schema_translate_map=None, ): if use_default_dialect: @@ -377,6 +380,9 @@ class AssertsCompiledSQL(object): if literal_binds: compile_kwargs["literal_binds"] = True + if render_postcompile: + compile_kwargs["render_postcompile"] = True + if isinstance(clause, orm.Query): context = clause._compile_context() context.statement.use_labels = True @@ -418,6 +424,22 @@ class AssertsCompiledSQL(object): eq_(tuple([p[x] for x in c.positiontup]), checkpositional) if check_prefetch is not None: eq_(c.prefetch, check_prefetch) + if check_literal_execute is not None: + eq_( + { + c.bind_names[b]: b.effective_value + for b in c.literal_execute_params + }, + check_literal_execute, + ) + if check_post_param is not None: + eq_( + { + c.bind_names[b]: b.effective_value + for b in c.post_compile_params + }, + check_post_param, + ) class ComparesTables(object): diff --git a/test/dialect/mssql/test_compiler.py b/test/dialect/mssql/test_compiler.py index b0f58563a7..9d46c3f355 100644 --- a/test/dialect/mssql/test_compiler.py +++ b/test/dialect/mssql/test_compiler.py @@ -304,7 +304,28 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): # "" # ) - def test_strict_binds(self): + @testing.combinations( + ( + lambda: select([literal("x"), literal("y")]), + "SELECT [POSTCOMPILE_param_1] AS anon_1, " + "[POSTCOMPILE_param_2] AS anon_2", + { + "check_literal_execute": {"param_1": "x", "param_2": "y"}, + "check_post_param": {}, + }, + ), + ( + lambda: select([t]).where(t.c.foo.in_(["x", "y", "z"])), + "SELECT sometable.foo FROM sometable WHERE sometable.foo " + "IN ([POSTCOMPILE_foo_1])", + { + "check_literal_execute": {"foo_1": ["x", "y", "z"]}, + "check_post_param": {}, + }, + ), + (lambda: t.c.foo.in_([None]), "sometable.foo IN (NULL)", {}), + ) + def test_strict_binds(self, expr, compiled, kw): """test the 'strict' compiler binds.""" from sqlalchemy.dialects.mssql.base import MSSQLStrictCompiler @@ -314,19 +335,8 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): t = table("sometable", column("foo")) - for expr, compiled in [ - ( - select([literal("x"), literal("y")]), - "SELECT 'x' AS anon_1, 'y' AS anon_2", - ), - ( - select([t]).where(t.c.foo.in_(["x", "y", "z"])), - "SELECT sometable.foo FROM sometable WHERE sometable.foo " - "IN ('x', 'y', 'z')", - ), - (t.c.foo.in_([None]), "sometable.foo IN (NULL)"), - ]: - self.assert_compile(expr, compiled, dialect=mxodbc_dialect) + expr = testing.resolve_lambda(expr, t=t) + self.assert_compile(expr, compiled, dialect=mxodbc_dialect, **kw) def test_in_with_subqueries(self): """Test removal of legacy behavior that converted "x==subquery" @@ -598,19 +608,26 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): self.assert_compile( u, "SELECT t1.col3 AS col3, t1.col4 AS col4 " - "FROM t1 WHERE t1.col2 IN (:col2_1, " - ":col2_2) UNION SELECT t2.col3 AS col3, " + "FROM t1 WHERE t1.col2 IN ([POSTCOMPILE_col2_1]) " + "UNION SELECT t2.col3 AS col3, " "t2.col4 AS col4 FROM t2 WHERE t2.col2 IN " - "(:col2_3, :col2_4) ORDER BY col3, col4", + "([POSTCOMPILE_col2_2]) ORDER BY col3, col4", + checkparams={ + "col2_1": ["t1col2r1", "t1col2r2"], + "col2_2": ["t2col2r2", "t2col2r3"], + }, ) self.assert_compile( u.alias("bar").select(), "SELECT bar.col3, bar.col4 FROM (SELECT " "t1.col3 AS col3, t1.col4 AS col4 FROM t1 " - "WHERE t1.col2 IN (:col2_1, :col2_2) UNION " + "WHERE t1.col2 IN ([POSTCOMPILE_col2_1]) UNION " "SELECT t2.col3 AS col3, t2.col4 AS col4 " - "FROM t2 WHERE t2.col2 IN (:col2_3, " - ":col2_4)) AS bar", + "FROM t2 WHERE t2.col2 IN ([POSTCOMPILE_col2_2])) AS bar", + checkparams={ + "col2_1": ["t1col2r1", "t1col2r2"], + "col2_2": ["t2col2r2", "t2col2r3"], + }, ) def test_function(self): diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index 638d843344..ec88277a6c 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -1113,9 +1113,27 @@ class SQLTest(fixtures.TestBase, AssertsCompiledSQL): ) def test_in_tuple(self): - self.assert_compile( - tuple_(column("q"), column("p")).in_([(1, 2), (3, 4)]), - "(q, p) IN (VALUES (?, ?), (?, ?))", + compiled = ( + tuple_(column("q"), column("p")) + .in_([(1, 2), (3, 4)]) + .compile(dialect=sqlite.dialect()) + ) + eq_(str(compiled), "(q, p) IN ([POSTCOMPILE_param_1])") + eq_( + compiled._literal_execute_expanding_parameter( + "param_1", + compiled.binds["param_1"], + compiled.binds["param_1"].value, + ), + ( + [ + ("param_1_1_1", 1), + ("param_1_1_2", 2), + ("param_1_2_1", 3), + ("param_1_2_2", 4), + ], + "VALUES (?, ?), (?, ?)", + ), ) diff --git a/test/engine/test_deprecations.py b/test/engine/test_deprecations.py index b7c3cf52f8..6fbf1be5b6 100644 --- a/test/engine/test_deprecations.py +++ b/test/engine/test_deprecations.py @@ -59,6 +59,18 @@ class CreateEngineTest(fixtures.TestBase): strategy="threadlocal", ) + def test_empty_in_keyword(self): + with testing.expect_deprecated( + "The create_engine.empty_in_strategy keyword is deprecated, " + "and no longer has any effect." + ): + create_engine( + "postgresql://", + empty_in_strategy="static", + module=Mock(), + _initialize=False, + ) + class TransactionTest(fixtures.TestBase): __backend__ = True diff --git a/test/ext/declarative/test_mixin.py b/test/ext/declarative/test_mixin.py index df7dea77c4..bda08e02f3 100644 --- a/test/ext/declarative/test_mixin.py +++ b/test/ext/declarative/test_mixin.py @@ -1399,9 +1399,9 @@ class DeclarativeMixinPropertyTest( self.assert_compile( s.query(Derived.data_syn).filter(Derived.data_syn == "foo"), "SELECT test.data AS test_data FROM test WHERE test.data = " - ":data_1 AND test.type IN (:type_1)", + ":data_1 AND test.type IN ([POSTCOMPILE_type_1])", dialect="default", - checkparams={"type_1": "derived", "data_1": "foo"}, + checkparams={"type_1": ["derived"], "data_1": "foo"}, ) def test_column_in_mapper_args(self): diff --git a/test/ext/test_horizontal_shard.py b/test/ext/test_horizontal_shard.py index 6ea124268c..e292cb2b08 100644 --- a/test/ext/test_horizontal_shard.py +++ b/test/ext/test_horizontal_shard.py @@ -113,8 +113,8 @@ class ShardTest(object): if binary.operator == operators.eq: ids.append(shard_lookup[binary.right.value]) elif binary.operator == operators.in_op: - for bind in binary.right.clauses: - ids.append(shard_lookup[bind.value]) + for value in binary.right.value: + ids.append(shard_lookup[value]) if query._criterion is not None: FindContinent().traverse(query._criterion) diff --git a/test/orm/inheritance/test_relationship.py b/test/orm/inheritance/test_relationship.py index a96c5ef04a..72a05871e0 100644 --- a/test/orm/inheritance/test_relationship.py +++ b/test/orm/inheritance/test_relationship.py @@ -1913,7 +1913,7 @@ class JoinedloadOverWPolyAliased( joinedload(cls.links).joinedload(Link.child).joinedload(cls.links) ) if cls is self.classes.Sub1: - extra = " WHERE parent.type IN (:type_1)" + extra = " WHERE parent.type IN ([POSTCOMPILE_type_1])" else: extra = "" @@ -1943,7 +1943,7 @@ class JoinedloadOverWPolyAliased( ) if Link.child.property.mapper.class_ is self.classes.Sub1: - extra = "AND parent_1.type IN (:type_1) " + extra = "AND parent_1.type IN ([POSTCOMPILE_type_1]) " else: extra = "" diff --git a/test/orm/inheritance/test_single.py b/test/orm/inheritance/test_single.py index d0db76b215..b070e2848a 100644 --- a/test/orm/inheritance/test_single.py +++ b/test/orm/inheritance/test_single.py @@ -267,7 +267,7 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): "employees.engineer_info AS " "employees_engineer_info, employees.type " "AS employees_type FROM employees WHERE " - "employees.type IN (:type_1, :type_2)) AS " + "employees.type IN ([POSTCOMPILE_type_1])) AS " "anon_1", use_default_dialect=True, ) @@ -282,13 +282,13 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): sess.query(a1.employee_id).select_from(a1), "SELECT employees_1.employee_id AS employees_1_employee_id " "FROM employees AS employees_1 WHERE employees_1.type " - "IN (:type_1, :type_2)", + "IN ([POSTCOMPILE_type_1])", ) self.assert_compile( sess.query(literal("1")).select_from(a1), "SELECT :param_1 AS anon_1 FROM employees AS employees_1 " - "WHERE employees_1.type IN (:type_1, :type_2)", + "WHERE employees_1.type IN ([POSTCOMPILE_type_1])", ) def test_union_modifiers(self): @@ -311,7 +311,7 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): "employees.engineer_info AS employees_engineer_info, " "employees.type AS employees_type FROM employees " "WHERE employees.engineer_info = :engineer_info_1 " - "AND employees.type IN (:type_1, :type_2) " + "AND employees.type IN ([POSTCOMPILE_type_1]) " "%(token)s " "SELECT employees.employee_id AS employees_employee_id, " "employees.name AS employees_name, " @@ -319,7 +319,7 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): "employees.engineer_info AS employees_engineer_info, " "employees.type AS employees_type FROM employees " "WHERE employees.manager_data = :manager_data_1 " - "AND employees.type IN (:type_3)) AS anon_1" + "AND employees.type IN ([POSTCOMPILE_type_2])) AS anon_1" ) for meth, token in [ @@ -334,11 +334,10 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): meth(q2), assert_sql % {"token": token}, checkparams={ - "manager_data_1": "bar", - "type_2": "juniorengineer", - "type_3": "manager", "engineer_info_1": "foo", - "type_1": "engineer", + "type_1": ["engineer", "juniorengineer"], + "manager_data_1": "bar", + "type_2": ["manager"], }, ) @@ -352,7 +351,7 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): "SELECT count(*) AS count_1 " "FROM (SELECT employees.employee_id AS employees_employee_id " "FROM employees " - "WHERE employees.type IN (:type_1, :type_2)) AS anon_1", + "WHERE employees.type IN ([POSTCOMPILE_type_1])) AS anon_1", use_default_dialect=True, ) @@ -422,7 +421,7 @@ class SingleInheritanceTest(testing.AssertsCompiledSQL, fixtures.MappedTest): ), "SELECT EXISTS (SELECT 1 FROM employees WHERE " "employees.name = :name_1 AND employees.type " - "IN (:type_1, :type_2)) AS anon_1", + "IN ([POSTCOMPILE_type_1])) AS anon_1", ) def test_type_filtering(self): @@ -566,7 +565,7 @@ class RelationshipFromSingleTest( "employee_stuff_name, anon_1.employee_id " "AS anon_1_employee_id FROM (SELECT " "employee.id AS employee_id FROM employee " - "WHERE employee.type IN (:type_1)) AS anon_1 " + "WHERE employee.type IN ([POSTCOMPILE_type_1])) AS anon_1 " "JOIN employee_stuff ON anon_1.employee_id " "= employee_stuff.employee_id ORDER BY " "anon_1.employee_id", @@ -713,7 +712,7 @@ class RelationshipToSingleTest( "companies.name AS companies_name FROM companies " "LEFT OUTER JOIN employees AS employees_1 ON " "companies.company_id = employees_1.company_id " - "AND employees_1.type IN (:type_1)", + "AND employees_1.type IN ([POSTCOMPILE_type_1])", ) def test_join_explicit_onclause_no_discriminator(self): @@ -768,7 +767,8 @@ class RelationshipToSingleTest( "companies.name AS companies_name, " "employees.name AS employees_name " "FROM companies LEFT OUTER JOIN employees ON companies.company_id " - "= employees.company_id AND employees.type IN (:type_1)", + "= employees.company_id " + "AND employees.type IN ([POSTCOMPILE_type_1])", ) def test_outer_join_prop_alias(self): @@ -797,7 +797,8 @@ class RelationshipToSingleTest( "companies.name AS companies_name, employees_1.name AS " "employees_1_name FROM companies LEFT OUTER " "JOIN employees AS employees_1 ON companies.company_id " - "= employees_1.company_id AND employees_1.type IN (:type_1)", + "= employees_1.company_id " + "AND employees_1.type IN ([POSTCOMPILE_type_1])", ) def test_outer_join_literal_onclause(self): @@ -831,7 +832,7 @@ class RelationshipToSingleTest( "employees.company_id AS employees_company_id FROM companies " "LEFT OUTER JOIN employees ON " "companies.company_id = employees.company_id " - "AND employees.type IN (:type_1)", + "AND employees.type IN ([POSTCOMPILE_type_1])", ) def test_outer_join_literal_onclause_alias(self): @@ -866,7 +867,7 @@ class RelationshipToSingleTest( "employees_1.company_id AS employees_1_company_id " "FROM companies LEFT OUTER JOIN employees AS employees_1 ON " "companies.company_id = employees_1.company_id " - "AND employees_1.type IN (:type_1)", + "AND employees_1.type IN ([POSTCOMPILE_type_1])", ) def test_outer_join_no_onclause(self): @@ -898,7 +899,7 @@ class RelationshipToSingleTest( "employees.company_id AS employees_company_id " "FROM companies LEFT OUTER JOIN employees ON " "companies.company_id = employees.company_id " - "AND employees.type IN (:type_1)", + "AND employees.type IN ([POSTCOMPILE_type_1])", ) def test_outer_join_no_onclause_alias(self): @@ -931,7 +932,7 @@ class RelationshipToSingleTest( "employees_1.company_id AS employees_1_company_id " "FROM companies LEFT OUTER JOIN employees AS employees_1 ON " "companies.company_id = employees_1.company_id " - "AND employees_1.type IN (:type_1)", + "AND employees_1.type IN ([POSTCOMPILE_type_1])", ) def test_correlated_column_select(self): @@ -965,7 +966,8 @@ class RelationshipToSingleTest( "SELECT companies.company_id AS companies_company_id, " "(SELECT count(employees.employee_id) AS count_1 " "FROM employees WHERE employees.company_id = " - "companies.company_id AND employees.type IN (:type_1)) AS anon_1 " + "companies.company_id " + "AND employees.type IN ([POSTCOMPILE_type_1])) AS anon_1 " "FROM companies", ) @@ -1041,8 +1043,8 @@ class RelationshipToSingleTest( "ON companies.company_id = employees.company_id " "JOIN employees " "ON companies.company_id = employees.company_id " - "AND employees.type IN (:type_1) " - "WHERE employees.type IN (:type_2)", + "AND employees.type IN ([POSTCOMPILE_type_1]) " + "WHERE employees.type IN ([POSTCOMPILE_type_2])", ) def test_relationship_to_subclass(self): @@ -1307,7 +1309,7 @@ class ManyToManyToSingleTest(fixtures.MappedTest, AssertsCompiledSQL): "child.name AS child_name " "FROM parent LEFT OUTER JOIN (m2m AS m2m_1 " "JOIN child ON child.id = m2m_1.child_id " - "AND child.discriminator IN (:discriminator_1)) " + "AND child.discriminator IN ([POSTCOMPILE_discriminator_1])) " "ON parent.id = m2m_1.parent_id", ) @@ -1324,7 +1326,8 @@ class ManyToManyToSingleTest(fixtures.MappedTest, AssertsCompiledSQL): "FROM parent LEFT OUTER JOIN " "(m2m AS m2m_1 JOIN child AS child_1 " "ON child_1.id = m2m_1.child_id AND child_1.discriminator " - "IN (:discriminator_1)) ON parent.id = m2m_1.parent_id", + "IN ([POSTCOMPILE_discriminator_1])) " + "ON parent.id = m2m_1.parent_id", ) @@ -1536,7 +1539,7 @@ class SingleFromPolySelectableTest( "engineer.manager_id AS engineer_manager_id " "FROM employee JOIN engineer ON employee.id = engineer.id) " "AS anon_1 " - "WHERE anon_1.employee_type IN (:type_1)", + "WHERE anon_1.employee_type IN ([POSTCOMPILE_type_1])", ) def test_query_wpoly_single_inh_subclass(self): @@ -1563,7 +1566,7 @@ class SingleFromPolySelectableTest( "engineer.engineer_info AS engineer_engineer_info, " "engineer.manager_id AS engineer_manager_id " "FROM employee JOIN engineer ON employee.id = engineer.id) " - "AS anon_1 WHERE anon_1.employee_type IN (:type_1)", + "AS anon_1 WHERE anon_1.employee_type IN ([POSTCOMPILE_type_1])", ) def test_single_inh_subclass_join_joined_inh_subclass(self): @@ -1582,7 +1585,7 @@ class SingleFromPolySelectableTest( "JOIN (employee AS employee_1 JOIN engineer AS engineer_1 " "ON employee_1.id = engineer_1.id) " "ON engineer_1.manager_id = manager.id " - "WHERE employee.type IN (:type_1)", + "WHERE employee.type IN ([POSTCOMPILE_type_1])", ) def test_single_inh_subclass_join_wpoly_joined_inh_subclass(self): @@ -1619,7 +1622,7 @@ class SingleFromPolySelectableTest( "FROM employee " "JOIN engineer ON employee.id = engineer.id) AS anon_1 " "ON anon_1.manager_id = manager.id " - "WHERE employee.type IN (:type_1)", + "WHERE employee.type IN ([POSTCOMPILE_type_1])", ) def test_joined_inh_subclass_join_single_inh_subclass(self): @@ -1639,7 +1642,7 @@ class SingleFromPolySelectableTest( "JOIN (employee AS employee_1 JOIN manager AS manager_1 " "ON employee_1.id = manager_1.id) " "ON engineer.manager_id = manager_1.id " - "AND employee_1.type IN (:type_1)", + "AND employee_1.type IN ([POSTCOMPILE_type_1])", ) diff --git a/test/orm/test_deferred.py b/test/orm/test_deferred.py index 0b95c324d7..f55eefc4ac 100644 --- a/test/orm/test_deferred.py +++ b/test/orm/test_deferred.py @@ -1129,8 +1129,8 @@ class DeferredOptionsTest(AssertsCompiledSQL, _fixtures.FixtureTest): expected = [ ( "SELECT users.id AS users_id, users.name AS users_name " - "FROM users WHERE users.id IN (:id_1, :id_2)", - {"id_2": 8, "id_1": 7}, + "FROM users WHERE users.id IN ([POSTCOMPILE_id_1])", + {"id_1": [7, 8]}, ), ( "SELECT addresses.id AS addresses_id, " diff --git a/test/orm/test_froms.py b/test/orm/test_froms.py index 736e27c143..b22bb1601a 100644 --- a/test/orm/test_froms.py +++ b/test/orm/test_froms.py @@ -2479,7 +2479,7 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): sess.query(User).join(sel, User.id > sel.c.id), "SELECT users.id AS users_id, users.name AS users_name FROM " "users JOIN (SELECT users.id AS id, users.name AS name FROM users " - "WHERE users.id IN (:id_1, :id_2)) " + "WHERE users.id IN ([POSTCOMPILE_id_1])) " "AS anon_1 ON users.id > anon_1.id", ) @@ -2490,8 +2490,9 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): "SELECT users_1.id AS users_1_id, users_1.name AS users_1_name " "FROM users AS users_1, (" "SELECT users.id AS id, users.name AS name FROM users " - "WHERE users.id IN (:id_1, :id_2)) AS anon_1 " + "WHERE users.id IN ([POSTCOMPILE_id_1])) AS anon_1 " "WHERE users_1.id > anon_1.id", + check_post_param={"id_1": [7, 8]}, ) self.assert_compile( @@ -2500,8 +2501,9 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): .join(ualias, ualias.id > sel.c.id), "SELECT users_1.id AS users_1_id, users_1.name AS users_1_name " "FROM (SELECT users.id AS id, users.name AS name " - "FROM users WHERE users.id IN (:id_1, :id_2)) AS anon_1 " + "FROM users WHERE users.id IN ([POSTCOMPILE_id_1])) AS anon_1 " "JOIN users AS users_1 ON users_1.id > anon_1.id", + check_post_param={"id_1": [7, 8]}, ) self.assert_compile( @@ -2510,8 +2512,9 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): .join(ualias, ualias.id > User.id), "SELECT users_1.id AS users_1_id, users_1.name AS users_1_name " "FROM (SELECT users.id AS id, users.name AS name FROM " - "users WHERE users.id IN (:id_1, :id_2)) AS anon_1 " + "users WHERE users.id IN ([POSTCOMPILE_id_1])) AS anon_1 " "JOIN users AS users_1 ON users_1.id > anon_1.id", + check_post_param={"id_1": [7, 8]}, ) salias = aliased(User, sel) @@ -2519,8 +2522,9 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): sess.query(salias).join(ualias, ualias.id > salias.id), "SELECT anon_1.id AS anon_1_id, anon_1.name AS anon_1_name FROM " "(SELECT users.id AS id, users.name AS name " - "FROM users WHERE users.id IN (:id_1, :id_2)) AS anon_1 " + "FROM users WHERE users.id IN ([POSTCOMPILE_id_1])) AS anon_1 " "JOIN users AS users_1 ON users_1.id > anon_1.id", + check_post_param={"id_1": [7, 8]}, ) self.assert_compile( @@ -2531,8 +2535,9 @@ class SelectFromTest(QueryTest, AssertsCompiledSQL): "FROM " "(SELECT users.id AS id, users.name AS name " "FROM users WHERE users.id " - "IN (:id_1, :id_2)) AS anon_1 " + "IN ([POSTCOMPILE_id_1])) AS anon_1 " "JOIN users AS users_1 ON users_1.id > anon_1.id", + check_post_param={"id_1": [7, 8]}, ) def test_aliased_class_vs_nonaliased(self): diff --git a/test/orm/test_of_type.py b/test/orm/test_of_type.py index 53a759c443..a431d231f4 100644 --- a/test/orm/test_of_type.py +++ b/test/orm/test_of_type.py @@ -1003,9 +1003,10 @@ class SubclassRelationshipTest3( "c.id AS c_id, c.type AS c_type, c.b_id AS c_b_id, a.id AS a_id, " "a.type AS a_type " "FROM a LEFT OUTER JOIN b ON " - "a.id = b.a_id AND b.type IN (:type_1) " + "a.id = b.a_id AND b.type IN ([POSTCOMPILE_type_1]) " "LEFT OUTER JOIN c ON " - "b.id = c.b_id AND c.type IN (:type_2) WHERE a.type IN (:type_3)" + "b.id = c.b_id AND c.type IN ([POSTCOMPILE_type_2]) " + "WHERE a.type IN ([POSTCOMPILE_type_3])" ) _query2 = ( @@ -1013,10 +1014,10 @@ class SubclassRelationshipTest3( "ccc.id AS ccc_id, ccc.type AS ccc_type, ccc.b_id AS ccc_b_id, " "aaa.id AS aaa_id, aaa.type AS aaa_type " "FROM a AS aaa LEFT OUTER JOIN b AS bbb " - "ON aaa.id = bbb.a_id AND bbb.type IN (:type_1) " + "ON aaa.id = bbb.a_id AND bbb.type IN ([POSTCOMPILE_type_1]) " "LEFT OUTER JOIN c AS ccc ON " - "bbb.id = ccc.b_id AND ccc.type IN (:type_2) " - "WHERE aaa.type IN (:type_3)" + "bbb.id = ccc.b_id AND ccc.type IN ([POSTCOMPILE_type_2]) " + "WHERE aaa.type IN ([POSTCOMPILE_type_3])" ) _query3 = ( @@ -1024,10 +1025,10 @@ class SubclassRelationshipTest3( "c.id AS c_id, c.type AS c_type, c.b_id AS c_b_id, " "aaa.id AS aaa_id, aaa.type AS aaa_type " "FROM a AS aaa LEFT OUTER JOIN b AS bbb " - "ON aaa.id = bbb.a_id AND bbb.type IN (:type_1) " + "ON aaa.id = bbb.a_id AND bbb.type IN ([POSTCOMPILE_type_1]) " "LEFT OUTER JOIN c ON " - "bbb.id = c.b_id AND c.type IN (:type_2) " - "WHERE aaa.type IN (:type_3)" + "bbb.id = c.b_id AND c.type IN ([POSTCOMPILE_type_2]) " + "WHERE aaa.type IN ([POSTCOMPILE_type_3])" ) def _test(self, join_of_type, of_type_for_c1, aliased_): diff --git a/test/orm/test_query.py b/test/orm/test_query.py index 068a7c8937..558b4d91c7 100644 --- a/test/orm/test_query.py +++ b/test/orm/test_query.py @@ -1756,7 +1756,7 @@ class OperatorTest(QueryTest, AssertsCompiledSQL): def test_in(self): User = self.classes.User - self._test(User.id.in_(["a", "b"]), "users.id IN (:id_1, :id_2)") + self._test(User.id.in_(["a", "b"]), "users.id IN ([POSTCOMPILE_id_1])") def test_in_on_relationship_not_supported(self): User, Address = self.classes.User, self.classes.Address diff --git a/test/orm/test_selectin_relations.py b/test/orm/test_selectin_relations.py index 2004923b63..9ff60d9252 100644 --- a/test/orm/test_selectin_relations.py +++ b/test/orm/test_selectin_relations.py @@ -3031,8 +3031,8 @@ class SingleInhSubclassTest( q.all, CompiledSQL( 'SELECT "user".id AS user_id, "user".type AS user_type ' - 'FROM "user" WHERE "user".type IN (:type_1)', - {"type_1": "employer"}, + 'FROM "user" WHERE "user".type IN ([POSTCOMPILE_type_1])', + {"type_1": ["employer"]}, ), CompiledSQL( "SELECT role.user_id AS role_user_id, role.id AS role_id " @@ -3155,8 +3155,8 @@ class M2OWDegradeTest( q.all, CompiledSQL( "SELECT a.id AS a_id, a.b_id AS a_b_id, a.q AS a_q " - "FROM a WHERE a.id IN (:id_1, :id_2) ORDER BY a.id", - [{"id_1": 1, "id_2": 3}], + "FROM a WHERE a.id IN ([POSTCOMPILE_id_1]) ORDER BY a.id", + [{"id_1": [1, 3]}], ), CompiledSQL( "SELECT b.id AS b_id, b.x AS b_x, b.y AS b_y " @@ -3184,8 +3184,8 @@ class M2OWDegradeTest( q.all, CompiledSQL( "SELECT a.id AS a_id, a.q AS a_q " - "FROM a WHERE a.id IN (:id_1, :id_2) ORDER BY a.id", - [{"id_1": 1, "id_2": 3}], + "FROM a WHERE a.id IN ([POSTCOMPILE_id_1]) ORDER BY a.id", + [{"id_1": [1, 3]}], ), # in the very unlikely case that the the FK col on parent is # deferred, we degrade to the JOIN version so that we don't need to diff --git a/test/sql/test_compare.py b/test/sql/test_compare.py index 6a2d47c20b..2d0e393314 100644 --- a/test/sql/test_compare.py +++ b/test/sql/test_compare.py @@ -391,18 +391,17 @@ class CoreFixtures(object): dont_compare_values_fixtures = [ lambda: ( - # same number of params each time, so compare for IN - # with legacy behavior of bind for each value works + # note the in_(...) all have different column names becuase + # otherwise all IN expressions would compare as equivalent column("x").in_(random_choices(range(10), k=3)), - # expanding IN places the whole list into a single parameter - # so it can be of arbitrary length as well - column("x").in_( + column("y").in_( bindparam( "q", random_choices(range(10), k=random.randint(0, 7)), expanding=True, ) ), + column("z").in_(random_choices(range(10), k=random.randint(0, 7))), column("x") == random.randint(1, 10), ) ] diff --git a/test/sql/test_compiler.py b/test/sql/test_compiler.py index 54c3e2f6ed..b49fb455cd 100644 --- a/test/sql/test_compiler.py +++ b/test/sql/test_compiler.py @@ -3327,22 +3327,77 @@ class BindParameterTest(AssertsCompiledSQL, fixtures.TestBase): _group_number=2, ) - def test_tuple(self): + def test_tuple_expanding_in_no_values(self): + expr = tuple_(table1.c.myid, table1.c.name).in_( + [(1, "foo"), (5, "bar")] + ) self.assert_compile( - tuple_(table1.c.myid, table1.c.name).in_([(1, "foo"), (5, "bar")]), - "(mytable.myid, mytable.name) IN " - "((:param_1, :param_2), (:param_3, :param_4))", + expr, + "(mytable.myid, mytable.name) IN " "([POSTCOMPILE_param_1])", + checkparams={"param_1": [(1, "foo"), (5, "bar")]}, + check_post_param={"param_1": [(1, "foo"), (5, "bar")]}, + check_literal_execute={}, + ) + + compiled = expr.compile() + ( + to_update, + replacement_expr, + ) = compiled._literal_execute_expanding_parameter( + "param_1", expr.right, [(1, "foo"), (5, "bar")] + ) + eq_( + to_update, + [ + ("param_1_1_1", 1), + ("param_1_1_2", "foo"), + ("param_1_2_1", 5), + ("param_1_2_2", "bar"), + ], + ) + eq_( + replacement_expr, + "(:param_1_1_1, :param_1_1_2), (:param_1_2_1, :param_1_2_2)", ) + def test_tuple_expanding_in_values(self): + expr = tuple_(table1.c.myid, table1.c.name).in_( + [(1, "foo"), (5, "bar")] + ) dialect = default.DefaultDialect() dialect.tuple_in_values = True self.assert_compile( tuple_(table1.c.myid, table1.c.name).in_([(1, "foo"), (5, "bar")]), - "(mytable.myid, mytable.name) IN " - "(VALUES (:param_1, :param_2), (:param_3, :param_4))", + "(mytable.myid, mytable.name) IN " "([POSTCOMPILE_param_1])", dialect=dialect, + checkparams={"param_1": [(1, "foo"), (5, "bar")]}, + check_post_param={"param_1": [(1, "foo"), (5, "bar")]}, + check_literal_execute={}, + ) + + compiled = expr.compile(dialect=dialect) + ( + to_update, + replacement_expr, + ) = compiled._literal_execute_expanding_parameter( + "param_1", expr.right, [(1, "foo"), (5, "bar")] + ) + eq_( + to_update, + [ + ("param_1_1_1", 1), + ("param_1_1_2", "foo"), + ("param_1_2_1", 5), + ("param_1_2_2", "bar"), + ], + ) + eq_( + replacement_expr, + "VALUES (:param_1_1_1, :param_1_1_2), " + "(:param_1_2_1, :param_1_2_2)", ) + def test_tuple_clauselist_in(self): self.assert_compile( tuple_(table1.c.myid, table1.c.name).in_( [tuple_(table2.c.otherid, table2.c.othername)] @@ -3410,69 +3465,87 @@ class BindParameterTest(AssertsCompiledSQL, fixtures.TestBase): "OR mytable.myid = :myid_2 OR mytable.myid = :myid_3", ) - def test_render_binds_as_literal(self): - """test a compiler that renders binds inline into - SQL in the columns clause.""" - + @testing.fixture + def ansi_compiler_fixture(self): dialect = default.DefaultDialect() - class Compiler(dialect.statement_compiler): + class Compiler(compiler.StrSQLCompiler): ansi_bind_rules = True dialect.statement_compiler = Compiler - self.assert_compile( - select([literal("someliteral")]), - "SELECT 'someliteral' AS anon_1", - dialect=dialect, - ) + return dialect - self.assert_compile( + @testing.combinations( + ( + "one", + select([literal("someliteral")]), + "SELECT [POSTCOMPILE_param_1] AS anon_1", + dict( + check_literal_execute={"param_1": "someliteral"}, + check_post_param={}, + ), + ), + ( + "two", select([table1.c.myid + 3]), - "SELECT mytable.myid + 3 AS anon_1 FROM mytable", - dialect=dialect, - ) - - self.assert_compile( + "SELECT mytable.myid + [POSTCOMPILE_myid_1] " + "AS anon_1 FROM mytable", + dict(check_literal_execute={"myid_1": 3}, check_post_param={}), + ), + ( + "three", select([table1.c.myid.in_([4, 5, 6])]), - "SELECT mytable.myid IN (4, 5, 6) AS anon_1 FROM mytable", - dialect=dialect, - ) - - self.assert_compile( + "SELECT mytable.myid IN ([POSTCOMPILE_myid_1]) " + "AS anon_1 FROM mytable", + dict( + check_literal_execute={"myid_1": [4, 5, 6]}, + check_post_param={}, + ), + ), + ( + "four", select([func.mod(table1.c.myid, 5)]), - "SELECT mod(mytable.myid, 5) AS mod_1 FROM mytable", - dialect=dialect, - ) - - self.assert_compile( + "SELECT mod(mytable.myid, [POSTCOMPILE_mod_2]) " + "AS mod_1 FROM mytable", + dict(check_literal_execute={"mod_2": 5}, check_post_param={}), + ), + ( + "five", select([literal("foo").in_([])]), - "SELECT 1 != 1 AS anon_1", - dialect=dialect, - ) - - self.assert_compile( + "SELECT [POSTCOMPILE_param_1] IN ([POSTCOMPILE_param_2]) " + "AS anon_1", + dict( + check_literal_execute={"param_1": "foo", "param_2": []}, + check_post_param={}, + ), + ), + ( + "six", select([literal(util.b("foo"))]), - "SELECT 'foo' AS anon_1", - dialect=dialect, - ) - - # test callable - self.assert_compile( + "SELECT [POSTCOMPILE_param_1] AS anon_1", + dict( + check_literal_execute={"param_1": util.b("foo")}, + check_post_param={}, + ), + ), + ( + "seven", select([table1.c.myid == bindparam("foo", callable_=lambda: 5)]), - "SELECT mytable.myid = 5 AS anon_1 FROM mytable", - dialect=dialect, - ) - - empty_in_dialect = default.DefaultDialect(empty_in_strategy="dynamic") - empty_in_dialect.statement_compiler = Compiler + "SELECT mytable.myid = [POSTCOMPILE_foo] AS anon_1 FROM mytable", + dict(check_literal_execute={"foo": 5}, check_post_param={}), + ), + argnames="stmt, expected, kw", + id_="iaaa", + ) + def test_render_binds_as_literal( + self, ansi_compiler_fixture, stmt, expected, kw + ): + """test a compiler that renders binds inline into + SQL in the columns clause.""" - assert_raises_message( - exc.CompileError, - "Bind parameter 'foo' without a " - "renderable value not allowed here.", - bindparam("foo").in_([]).compile, - dialect=empty_in_dialect, + self.assert_compile( + stmt, expected, dialect=ansi_compiler_fixture, **kw ) def test_render_literal_execute_parameter(self): @@ -3493,6 +3566,15 @@ class BindParameterTest(AssertsCompiledSQL, fixtures.TestBase): literal_binds=True, ) + def test_render_literal_execute_parameter_render_postcompile(self): + self.assert_compile( + select([table1.c.myid]).where( + table1.c.myid == bindparam("foo", 5, literal_execute=True) + ), + "SELECT mytable.myid FROM mytable " "WHERE mytable.myid = 5", + render_postcompile=True, + ) + def test_render_expanding_parameter(self): self.assert_compile( select([table1.c.myid]).where( @@ -3512,6 +3594,20 @@ class BindParameterTest(AssertsCompiledSQL, fixtures.TestBase): literal_binds=True, ) + def test_render_expanding_parameter_render_postcompile(self): + # renders the IN the old way, essentially, but creates the bound + # parameters on the fly. + + self.assert_compile( + select([table1.c.myid]).where( + table1.c.myid.in_(bindparam("foo", [1, 2, 3], expanding=True)) + ), + "SELECT mytable.myid FROM mytable " + "WHERE mytable.myid IN (:foo_1, :foo_2, :foo_3)", + render_postcompile=True, + checkparams={"foo_1": 1, "foo_2": 2, "foo_3": 3}, + ) + class UnsupportedTest(fixtures.TestBase): def test_unsupported_element_str_visit_name(self): diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index 1e1835d1ac..aa56d0b6b3 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -45,7 +45,6 @@ from sqlalchemy.sql.elements import Label from sqlalchemy.sql.expression import BinaryExpression from sqlalchemy.sql.expression import ClauseList from sqlalchemy.sql.expression import func -from sqlalchemy.sql.expression import Grouping from sqlalchemy.sql.expression import select from sqlalchemy.sql.expression import tuple_ from sqlalchemy.sql.expression import UnaryExpression @@ -161,12 +160,8 @@ class DefaultColumnComparatorTest(fixtures.TestBase): assert left.comparator.operate(operators.in_op, [1, 2, 3]).compare( BinaryExpression( left, - Grouping( - ClauseList( - BindParameter("left", value=1, unique=True), - BindParameter("left", value=2, unique=True), - BindParameter("left", value=3, unique=True), - ) + BindParameter( + "left", value=[1, 2, 3], unique=True, expanding=True ), operators.in_op, ) @@ -178,12 +173,8 @@ class DefaultColumnComparatorTest(fixtures.TestBase): assert left.comparator.operate(operators.notin_op, [1, 2, 3]).compare( BinaryExpression( left, - Grouping( - ClauseList( - BindParameter("left", value=1, unique=True), - BindParameter("left", value=2, unique=True), - BindParameter("left", value=3, unique=True), - ) + BindParameter( + "left", value=[1, 2, 3], unique=True, expanding=True ), operators.notin_op, ) @@ -1204,14 +1195,14 @@ class OperatorPrecedenceTest(fixtures.TestBase, testing.AssertsCompiledSQL): self.assert_compile( self.table2.select(5 + self.table2.c.field.in_([5, 6])), "SELECT op.field FROM op WHERE :param_1 + " - "(op.field IN (:field_1, :field_2))", + "(op.field IN ([POSTCOMPILE_field_1]))", ) def test_operator_precedence_6(self): self.assert_compile( self.table2.select((5 + self.table2.c.field).in_([5, 6])), "SELECT op.field FROM op WHERE :field_1 + op.field " - "IN (:param_1, :param_2)", + "IN ([POSTCOMPILE_param_1])", ) def test_operator_precedence_7(self): @@ -1548,29 +1539,32 @@ class InTest(fixtures.TestBase, testing.AssertsCompiledSQL): "myothertable", column("otherid", Integer), column("othername", String) ) - def _dialect(self, empty_in_strategy="static"): - return default.DefaultDialect(empty_in_strategy=empty_in_strategy) - def test_in_1(self): self.assert_compile( - self.table1.c.myid.in_(["a"]), "mytable.myid IN (:myid_1)" + self.table1.c.myid.in_(["a"]), + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": ["a"]}, ) def test_in_2(self): self.assert_compile( - ~self.table1.c.myid.in_(["a"]), "mytable.myid NOT IN (:myid_1)" + ~self.table1.c.myid.in_(["a"]), + "mytable.myid NOT IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": ["a"]}, ) def test_in_3(self): self.assert_compile( self.table1.c.myid.in_(["a", "b"]), - "mytable.myid IN (:myid_1, :myid_2)", + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": ["a", "b"]}, ) def test_in_4(self): self.assert_compile( self.table1.c.myid.in_(iter(["a", "b"])), - "mytable.myid IN (:myid_1, :myid_2)", + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": ["a", "b"]}, ) def test_in_5(self): @@ -1604,6 +1598,8 @@ class InTest(fixtures.TestBase, testing.AssertsCompiledSQL): ) def test_in_10(self): + # when non-literal expressions are present we still need to do the + # old way where we render up front self.assert_compile( self.table1.c.myid.in_([literal("a") + "a", "b"]), "mytable.myid IN (:param_1 || :param_2, :myid_1)", @@ -1662,7 +1658,8 @@ class InTest(fixtures.TestBase, testing.AssertsCompiledSQL): def test_in_19(self): self.assert_compile( self.table1.c.myid.in_([1, 2, 3]), - "mytable.myid IN (:myid_1, :myid_2, :myid_3)", + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": [1, 2, 3]}, ) def test_in_20(self): @@ -1757,10 +1754,26 @@ class InTest(fixtures.TestBase, testing.AssertsCompiledSQL): self.table1.c.myid.in_([None]), "mytable.myid IN (NULL)" ) + def test_in_29(self): + a, b, c = ( + column("a", Integer), + column("b", String), + column("c", LargeBinary), + ) + t1 = tuple_(a, b, c) + expr = t1.in_([(3, "hi", "there"), (4, "Q", "P")]) + self.assert_compile( + expr, + "(a, b, c) IN ([POSTCOMPILE_param_1])", + checkparams={"param_1": [(3, "hi", "there"), (4, "Q", "P")]}, + ) + def test_in_set(self): + s = {1, 2, 3} self.assert_compile( - self.table1.c.myid.in_({1, 2, 3}), - "mytable.myid IN (:myid_1, :myid_2, :myid_3)", + self.table1.c.myid.in_(s), + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": list(s)}, ) def test_in_arbitrary_sequence(self): @@ -1777,69 +1790,10 @@ class InTest(fixtures.TestBase, testing.AssertsCompiledSQL): seq = MySeq([1, 2, 3]) self.assert_compile( self.table1.c.myid.in_(seq), - "mytable.myid IN (:myid_1, :myid_2, :myid_3)", - ) - - def test_empty_in_dynamic_1(self): - self.assert_compile( - self.table1.c.myid.in_([]), - "mytable.myid != mytable.myid", - dialect=self._dialect("dynamic"), + "mytable.myid IN ([POSTCOMPILE_myid_1])", + checkparams={"myid_1": [1, 2, 3]}, ) - def test_empty_in_dynamic_2(self): - self.assert_compile( - self.table1.c.myid.notin_([]), - "mytable.myid = mytable.myid", - dialect=self._dialect("dynamic"), - ) - - def test_empty_in_dynamic_3(self): - self.assert_compile( - ~self.table1.c.myid.in_([]), - "mytable.myid = mytable.myid", - dialect=self._dialect("dynamic"), - ) - - def test_empty_in_dynamic_warn_1(self): - with testing.expect_warnings( - "The IN-predicate was invoked with an empty sequence." - ): - self.assert_compile( - self.table1.c.myid.in_([]), - "mytable.myid != mytable.myid", - dialect=self._dialect("dynamic_warn"), - ) - - def test_empty_in_dynamic_warn_2(self): - with testing.expect_warnings( - "The IN-predicate was invoked with an empty sequence." - ): - self.assert_compile( - self.table1.c.myid.notin_([]), - "mytable.myid = mytable.myid", - dialect=self._dialect("dynamic_warn"), - ) - - def test_empty_in_dynamic_warn_3(self): - with testing.expect_warnings( - "The IN-predicate was invoked with an empty sequence." - ): - self.assert_compile( - ~self.table1.c.myid.in_([]), - "mytable.myid = mytable.myid", - dialect=self._dialect("dynamic_warn"), - ) - - def test_empty_in_static_1(self): - self.assert_compile(self.table1.c.myid.in_([]), "1 != 1") - - def test_empty_in_static_2(self): - self.assert_compile(self.table1.c.myid.notin_([]), "1 = 1") - - def test_empty_in_static_3(self): - self.assert_compile(~self.table1.c.myid.in_([]), "1 = 1") - class MathOperatorTest(fixtures.TestBase, testing.AssertsCompiledSQL): __dialect__ = "default" @@ -2709,9 +2663,9 @@ class CustomOpTest(fixtures.TestBase): class TupleTypingTest(fixtures.TestBase): def _assert_types(self, expr): - eq_(expr.clauses[0].type._type_affinity, Integer) - eq_(expr.clauses[1].type._type_affinity, String) - eq_(expr.clauses[2].type._type_affinity, LargeBinary()._type_affinity) + eq_(expr[0]._type_affinity, Integer) + eq_(expr[1]._type_affinity, String) + eq_(expr[2]._type_affinity, LargeBinary()._type_affinity) def test_type_coercion_on_eq(self): a, b, c = ( @@ -2721,7 +2675,7 @@ class TupleTypingTest(fixtures.TestBase): ) t1 = tuple_(a, b, c) expr = t1 == (3, "hi", "there") - self._assert_types(expr.right) + self._assert_types([bind.type for bind in expr.right.element.clauses]) def test_type_coercion_on_in(self): a, b, c = ( @@ -2731,9 +2685,9 @@ class TupleTypingTest(fixtures.TestBase): ) t1 = tuple_(a, b, c) expr = t1.in_([(3, "hi", "there"), (4, "Q", "P")]) - eq_(len(expr.right.clauses), 2) - for elem in expr.right.clauses: - self._assert_types(elem) + + eq_(len(expr.right.value), 2) + self._assert_types(expr.right._expanding_in_types) class AnyAllTest(fixtures.TestBase, testing.AssertsCompiledSQL): diff --git a/test/sql/test_query.py b/test/sql/test_query.py index 3b5e7dff2a..9e56b6489f 100644 --- a/test/sql/test_query.py +++ b/test/sql/test_query.py @@ -813,7 +813,7 @@ class QueryTest(fixtures.TestBase): users.insert().execute(user_id=8, user_name="fred") users.insert().execute(user_id=9, user_name=None) - u = bindparam("search_key") + u = bindparam("search_key", type_=String) s = users.select(not_(u.in_([]))) r = s.execute(search_key="john").fetchall() @@ -821,7 +821,6 @@ class QueryTest(fixtures.TestBase): r = s.execute(search_key=None).fetchall() assert len(r) == 3 - @testing.emits_warning(".*empty sequence.*") def test_literal_in(self): """similar to test_bind_in but use a bind with a value.""" @@ -861,40 +860,6 @@ class QueryTest(fixtures.TestBase): r = conn.execute(s).fetchall() assert len(r) == 0 - @testing.requires.boolean_col_expressions - def test_empty_in_filtering_dynamic(self): - """test the behavior of the in_() function when - comparing against an empty collection, specifically - that a proper boolean value is generated. - - """ - - engine = engines.testing_engine( - options={"empty_in_strategy": "dynamic"} - ) - - with engine.connect() as conn: - users.create(engine, checkfirst=True) - - conn.execute( - users.insert(), - [ - {"user_id": 7, "user_name": "jack"}, - {"user_id": 8, "user_name": "ed"}, - {"user_id": 9, "user_name": None}, - ], - ) - - s = users.select(users.c.user_name.in_([]) == True) # noqa - r = conn.execute(s).fetchall() - assert len(r) == 0 - s = users.select(users.c.user_name.in_([]) == False) # noqa - r = conn.execute(s).fetchall() - assert len(r) == 2 - s = users.select(users.c.user_name.in_([]) == None) # noqa - r = conn.execute(s).fetchall() - assert len(r) == 1 - class RequiredBindTest(fixtures.TablesTest): run_create_tables = None diff --git a/test/sql/test_selectable.py b/test/sql/test_selectable.py index 184e4a99c2..be7e28b5d3 100644 --- a/test/sql/test_selectable.py +++ b/test/sql/test_selectable.py @@ -2442,7 +2442,7 @@ class AnnotationsTest(fixtures.TestBase): (table1.c.col1 == 5, "table1.col1 = :col1_1"), ( table1.c.col1.in_([2, 3, 4]), - "table1.col1 IN (:col1_1, :col1_2, " ":col1_3)", + "table1.col1 IN ([POSTCOMPILE_col1_1])", ), ]: eq_(str(expr), expected)