From: Mike Bayer Date: Mon, 19 Jun 2017 20:35:53 +0000 (-0400) Subject: Add ad-hoc mapped expressions X-Git-Tag: rel_1_2_0b1~17 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=83c1e03c5c74c69facfc371840ffae890f05c338;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Add ad-hoc mapped expressions Added a new feature :func:`.orm.with_expression` that allows an ad-hoc SQL expression to be added to a specific entity in a query at result time. This is an alternative to the SQL expression being delivered as a separate element in the result tuple. Change-Id: Id8c479f7489fb02e09427837c59d1eabb2a6c014 Fixes: #3058 --- diff --git a/doc/build/changelog/changelog_12.rst b/doc/build/changelog/changelog_12.rst index 7e081f64de..a963b96f07 100644 --- a/doc/build/changelog/changelog_12.rst +++ b/doc/build/changelog/changelog_12.rst @@ -13,6 +13,19 @@ .. changelog:: :version: 1.2.0b1 + .. change:: 3058 + :tags: feature, orm + :tickets: 3058 + + Added a new feature :func:`.orm.with_expression` that allows an ad-hoc + SQL expression to be added to a specific entity in a query at result + time. This is an alternative to the SQL expression being delivered as + a separate element in the result tuple. + + .. seealso:: + + :ref:`change_3058` + .. change:: 3496 :tags: bug, orm :tickets: 3496 diff --git a/doc/build/changelog/migration_12.rst b/doc/build/changelog/migration_12.rst index 6cc955a66f..f0857c5314 100644 --- a/doc/build/changelog/migration_12.rst +++ b/doc/build/changelog/migration_12.rst @@ -213,6 +213,39 @@ are loaded with additional SELECT statements: :ticket:`3948` +.. _change_3058: + +ORM attributes that can receive ad-hoc SQL expressions +------------------------------------------------------ + +A new ORM attribute type :func:`.orm.deferred_expression` is added which +is similar to :func:`.orm.deferred`, except its SQL expression +is determined at query time using a new option :func:`.orm.with_expression`; +if not specified, the attribute defaults to ``None``:: + + from sqlalchemy.orm import deferred_expression + from sqlalchemy.orm import with_expression + + class A(Base): + __tablename__ = 'a' + id = Column(Integer, primary_key=True) + x = Column(Integer) + y = Column(Integer) + + # will be None normally... + expr = deferred_expression() + + # but let's give it x + y + a1 = session.query(A).options( + with_expression(A.expr, A.x + A.y)).first() + print(a1.expr) + +.. seealso:: + + :ref:`mapper_deferred_expression` + +:ticket:`3058` + .. _change_3229: Support for bulk updates of hybrids, composites diff --git a/doc/build/orm/loading_columns.rst b/doc/build/orm/loading_columns.rst index afa54078f1..289ba5c332 100644 --- a/doc/build/orm/loading_columns.rst +++ b/doc/build/orm/loading_columns.rst @@ -132,9 +132,11 @@ unchanged, use :func:`.orm.defaultload`:: Column Deferral API ------------------- +.. autofunction:: defer + .. autofunction:: deferred -.. autofunction:: defer +.. autofunction:: deferred_expression .. autofunction:: load_only @@ -142,6 +144,8 @@ Column Deferral API .. autofunction:: undefer_group +.. autofunction:: with_expression + .. _bundles: Column Bundles diff --git a/doc/build/orm/mapped_sql_expr.rst b/doc/build/orm/mapped_sql_expr.rst index 858e6973e1..7d0481d9cc 100644 --- a/doc/build/orm/mapped_sql_expr.rst +++ b/doc/build/orm/mapped_sql_expr.rst @@ -206,3 +206,65 @@ The plain descriptor approach is useful as a last resort, but is less performant in the usual case than both the hybrid and column property approaches, in that it needs to emit a SQL query upon each access. +.. _mapper_deferred_expression: + +Query-time SQL expressions as mapped attributes +----------------------------------------------- + +When using :meth:`.Session.query`, we have the option to specify not just +mapped entities but ad-hoc SQL expressions as well. Suppose if a class +``A`` had integer attributes ``.x`` and ``.y``, we could query for ``A`` +objects, and additionally the sum of ``.x`` and ``.y``, as follows:: + + q = session.query(A, A.x + A.y) + +The above query returns tuples of the form ``(A object, integer)``. + +An option exists which can apply the ad-hoc ``A.x + A.y`` expression to the +returned ``A`` objects instead of as a separate tuple entry; this is the +:func:`.with_expression` query option in conjunction with the +:func:`.deferred_expression` attribute mapping. The class is mapped +to include a placeholder attribute where any particular SQL expression +may be applied:: + + from sqlalchemy.orm import deferred_expression + + class A(Base): + __tablename__ = 'a' + id = Column(Integer, primary_key=True) + x = Column(Integer) + y = Column(Integer) + + expr = deferred_expression() + +We can then query for objects of type ``A``, applying an arbitrary +SQL expression to be populated into ``A.expr``:: + + from sqlalchemy.orm import with_expression + q = session.query(A).options( + with_expression(A.expr, A.x + A.y)) + +The :func:`.deferred_expression` mapping has these caveats: + +* On an object where :func:`.deferred_expression` were not used to populate + the attribute, the attribute on an object instance will have the value + ``None``. + +* The mapped attribute currently **cannot** be applied to other parts of the + query and make use of the ad-hoc expression; that is, this won't work:: + + q = session.query(A).options( + with_expression(A.expr, A.x + A.y) + ).filter(A.expr > 5) + + The ``A.expr`` expression will resolve to NULL in the above WHERE clause. + To use the expression throughout the query, assign to a variable and + use that:: + + a_expr = A.x + A.y + q = session.query(A).options( + with_expression(A.expr, a_expr) + ).filter(a_expr > 5) + +.. versionadded:: 1.2 + diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index 7ecd5b67eb..80c27e392d 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -68,6 +68,7 @@ from .query import AliasOption, Query, Bundle from ..util.langhelpers import public_factory from .. import util as _sa_util from . import strategies as _strategies +from .. import sql as _sql def create_session(bind=None, **kwargs): @@ -177,6 +178,20 @@ def deferred(*columns, **kw): return ColumnProperty(deferred=True, *columns, **kw) +def deferred_expression(): + """Indicate an attribute that populates from a query-time SQL expression. + + .. versionadded:: 1.2 + + .. seealso:: + + :ref:`mapper_deferred_expression` + + """ + prop = ColumnProperty(_sql.null()) + prop.strategy_key = (("deferred_expression", True),) + return prop + mapper = public_factory(Mapper, ".orm.mapper") synonym = public_factory(SynonymProperty, ".orm.synonym") @@ -235,6 +250,7 @@ contains_eager = strategy_options.contains_eager._unbound_fn defer = strategy_options.defer._unbound_fn undefer = strategy_options.undefer._unbound_fn undefer_group = strategy_options.undefer_group._unbound_fn +with_expression = strategy_options.with_expression._unbound_fn load_only = strategy_options.load_only._unbound_fn lazyload = strategy_options.lazyload._unbound_fn lazyload_all = strategy_options.lazyload_all._unbound_all_fn diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 690984a6a7..4e7636cf83 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -194,6 +194,48 @@ class ColumnLoader(LoaderStrategy): populators["expire"].append((self.key, True)) +@log.class_logger +@properties.ColumnProperty.strategy_for(deferred_expression=True) +class ExpressionColumnLoader(ColumnLoader): + def __init__(self, parent, strategy_key): + super(ExpressionColumnLoader, self).__init__(parent, strategy_key) + + def setup_query( + self, context, entity, path, loadopt, + adapter, column_collection, memoized_populators, **kwargs): + + if loadopt and "expression" in loadopt.local_opts: + columns = [loadopt.local_opts["expression"]] + + for c in columns: + if adapter: + c = adapter.columns[c] + column_collection.append(c) + + fetch = columns[0] + if adapter: + fetch = adapter.columns[fetch] + memoized_populators[self.parent_property] = fetch + + def create_row_processor( + self, context, path, + loadopt, mapper, result, adapter, populators): + # look through list of columns represented here + # to see which, if any, is present in the row. + if loadopt and "expression" in loadopt.local_opts: + columns = [loadopt.local_opts["expression"]] + + for col in columns: + if adapter: + col = adapter.columns[col] + getter = result._getter(col, False) + if getter: + populators["quick"].append((self.key, getter)) + break + else: + populators["expire"].append((self.key, True)) + + @log.class_logger @properties.ColumnProperty.strategy_for(deferred=True, instrument=True) @properties.ColumnProperty.strategy_for(do_nothing=True) diff --git a/lib/sqlalchemy/orm/strategy_options.py b/lib/sqlalchemy/orm/strategy_options.py index 4c7e34ebbf..2cdf6ba957 100644 --- a/lib/sqlalchemy/orm/strategy_options.py +++ b/lib/sqlalchemy/orm/strategy_options.py @@ -1348,6 +1348,54 @@ def undefer_group(name): return _UnboundLoad().undefer_group(name) +from ..sql import expression as sql_expr +from .util import _orm_full_deannotate + + +@loader_option() +def with_expression(loadopt, key, expression): + r"""Apply an ad-hoc SQL expression to a "deferred expression" attribute. + + This option is used in conjunction with the :func:`.orm.deferred_expression` + mapper-level construct that indicates an attribute which should be the + target of an ad-hoc SQL expression. + + E.g.:: + + + sess.query(SomeClass).options( + with_expression(SomeClass.x_y_expr, SomeClass.x + SomeClass.y) + ) + + .. versionadded:: 1.2 + + :param key: Attribute to be undeferred. + + :param expr: SQL expression to be applied to the attribute. + + .. seealso:: + + :ref:`mapper_deferred_expression` + + """ + + expression = sql_expr._labeled( + _orm_full_deannotate(expression)) + + return loadopt.set_column_strategy( + (key, ), + {"deferred_expression": True}, + opts={"expression": expression} + ) + + +@with_expression._add_unbound_fn +def with_expression(key, expression): + return _UnboundLoad._from_keys( + _UnboundLoad.with_expression, (key, ), + False, {"expression": expression}) + + @loader_option() def selectin_polymorphic(loadopt, classes): """Indicate an eager load should take place for all attributes diff --git a/test/orm/test_deferred.py b/test/orm/test_deferred.py index 4b5eafffed..9c2dc4c8b7 100644 --- a/test/orm/test_deferred.py +++ b/test/orm/test_deferred.py @@ -2,9 +2,13 @@ import sqlalchemy as sa from sqlalchemy import testing, util from sqlalchemy.orm import mapper, deferred, defer, undefer, Load, \ load_only, undefer_group, create_session, synonym, relationship, Session,\ - joinedload, defaultload, aliased, contains_eager, with_polymorphic + joinedload, defaultload, aliased, contains_eager, with_polymorphic, \ + deferred_expression, with_expression from sqlalchemy.testing import eq_, AssertsCompiledSQL, assert_raises_message from test.orm import _fixtures +from sqlalchemy.testing.schema import Column +from sqlalchemy import Integer, ForeignKey +from sqlalchemy.testing import fixtures from .inheritance._poly_fixtures import Company, Person, Engineer, Manager, \ @@ -868,3 +872,104 @@ class InheritanceTest(_Polymorphic): "ON people.person_id = managers.person_id " "ORDER BY people.person_id" ) + + + +class WithExpressionTest(fixtures.DeclarativeMappedTest): + @classmethod + def setup_classes(cls): + Base = cls.DeclarativeBasic + + class A(fixtures.ComparableEntity, Base): + __tablename__ = 'a' + id = Column(Integer, primary_key=True) + x = Column(Integer) + y = Column(Integer) + + my_expr = deferred_expression() + + bs = relationship("B", order_by="B.id") + + class B(fixtures.ComparableEntity, Base): + __tablename__ = 'b' + id = Column(Integer, primary_key=True) + a_id = Column(ForeignKey('a.id')) + p = Column(Integer) + q = Column(Integer) + + b_expr = deferred_expression() + + @classmethod + def insert_data(cls): + A, B = cls.classes("A", "B") + s = Session() + + s.add_all([ + A(id=1, x=1, y=2, bs=[B(id=1, p=1, q=2), B(id=2, p=4, q=8)]), + A(id=2, x=2, y=3), + A(id=3, x=5, y=10, bs=[B(id=3, p=5, q=0)]), + A(id=4, x=2, y=10, bs=[B(id=4, p=19, q=8), B(id=5, p=5, q=5)]), + ]) + + s.commit() + + def test_simple_expr(self): + A = self.classes.A + + s = Session() + a1 = s.query(A).options( + with_expression(A.my_expr, A.x + A.y)).filter(A.x > 1).\ + order_by(A.id) + + eq_( + a1.all(), + [ + A(my_expr=5), A(my_expr=15), A(my_expr=12) + ] + ) + + def test_reuse_expr(self): + A = self.classes.A + + s = Session() + + # so people will obv. want to say, "filter(A.my_expr > 10)". + # but that means Query or Core has to post-modify the statement + # after construction. + expr = A.x + A.y + a1 = s.query(A).options( + with_expression(A.my_expr, expr)).filter(expr > 10).\ + order_by(expr) + + eq_( + a1.all(), + [A(my_expr=12), A(my_expr=15)] + ) + + def test_in_joinedload(self): + A, B = self.classes("A", "B") + + s = Session() + + q = s.query(A).options( + joinedload(A.bs).with_expression(B.b_expr, B.p * A.x) + ).filter(A.id.in_([3, 4])).order_by(A.id) + + eq_( + q.all(), + [ + A(bs=[B(b_expr=25)]), + A(bs=[B(b_expr=38), B(b_expr=10)]) + ] + ) + + def test_no_sql_not_set_up(self): + A = self.classes.A + + s = Session() + a1 = s.query(A).first() + + def go(): + eq_(a1.my_expr, None) + + self.assert_sql_count(testing.db, go, 0) \ No newline at end of file