From 912fb6c2d54d7f2fcda654a8f7702d122e8b8d70 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 17 May 2017 13:05:04 -0400 Subject: [PATCH] Add new configuration, inspection for baked queries Added new flag :paramref:`.Session.enable_baked_queries` to the :class:`.Session` to allow baked queries to be disabled session-wide, reducing memory use. Also added new :class:`.Bakery` wrapper so that the bakery returned by :paramref:`.BakedQuery.bakery` can be inspected. Change-Id: I5657af7a99d2b24c89d6aee1343f432728e3f807 --- doc/build/changelog/changelog_12.rst | 9 +++++++ doc/build/orm/extensions/baked.rst | 30 +++++++++++++++++++++++- lib/sqlalchemy/ext/baked.py | 32 ++++++++++++++++++++----- lib/sqlalchemy/orm/session.py | 17 ++++++++++++++ test/ext/test_baked.py | 35 ++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 7 deletions(-) diff --git a/doc/build/changelog/changelog_12.rst b/doc/build/changelog/changelog_12.rst index 3d0b7fa088..d8efd74a9a 100644 --- a/doc/build/changelog/changelog_12.rst +++ b/doc/build/changelog/changelog_12.rst @@ -13,6 +13,15 @@ .. changelog:: :version: 1.2.0b1 + .. change:: baked_opts + :tags: feature, ext + + Added new flag :paramref:`.Session.enable_baked_queries` to the + :class:`.Session` to allow baked queries to be disabled + session-wide, reducing memory use. Also added new :class:`.Bakery` + wrapper so that the bakery returned by :paramref:`.BakedQuery.bakery` + can be inspected. + .. change:: 3988 :tags: bug, orm :tickets: 3988 diff --git a/doc/build/orm/extensions/baked.rst b/doc/build/orm/extensions/baked.rst index 0dceedc4a1..593293b460 100644 --- a/doc/build/orm/extensions/baked.rst +++ b/doc/build/orm/extensions/baked.rst @@ -332,6 +332,27 @@ to arrive at the current "baked" approach. Starting from the management, removal of all redundant Python execution, and queries built up with conditionals needed to be addressed, leading to the final approach. +Disabling Baked Queries Session-wide +------------------------------------ + +The flag :paramref:`.Session.enable_baked_queries` may be set to False, +causing all baked queries to not use the cache when used against that +:class:`.Session`:: + + session = Session(engine, enable_baked_queries=False) + +Like all session flags, it is also accepted by factory objects like +:class:`.sessionmaker` and methods like :meth:`.sessionmaker.configure`. + +The immediate rationale for this flag is to reduce memory use in the case +that the query baking used by relationship loaders and other loaders +is not desirable. It also can be used in the case that an application +which is seeing issues potentially due to cache key conflicts from user-defined +baked queries or other baked query issues can turn the behavior off, in +order to identify or eliminate baked queries as the cause of an issue. + +.. versionadded:: 1.2 + Lazy Loading Integration ------------------------ @@ -350,7 +371,11 @@ Opting out with the bake_queries flag The :func:`.relationship` construct includes a flag :paramref:`.relationship.bake_queries` which when set to False will cause -that relationship to opt out of caching queries. +that relationship to opt out of caching queries. Additionally, the +:paramref:`.Session.enable_baked_queries` setting can be used to disable +all "baked query" use. These flags can be useful to conserve memory, +when memory conservation is more important than performance for a particular +relationship or for the application overall. API Documentation ----------------- @@ -360,6 +385,9 @@ API Documentation .. autoclass:: BakedQuery :members: +.. autoclass:: Bakery + :members: + .. autoclass:: Result :members: diff --git a/lib/sqlalchemy/ext/baked.py b/lib/sqlalchemy/ext/baked.py index 95b618f3f5..ba3c2aed04 100644 --- a/lib/sqlalchemy/ext/baked.py +++ b/lib/sqlalchemy/ext/baked.py @@ -28,6 +28,27 @@ import logging log = logging.getLogger(__name__) +class Bakery(object): + """Callable which returns a :class:`.BakedQuery`. + + This object is returned by the class method + :meth:`.BakedQuery.bakery`. It exists as an object + so that the "cache" can be easily inspected. + + .. versionadded:: 1.2 + + + """ + __slots__ = 'cls', 'cache' + + def __init__(self, cls_, cache): + self.cls = cls_ + self.cache = cache + + def __call__(self, initial_fn, *args): + return self.cls(self.cache, initial_fn, args) + + class BakedQuery(object): """A builder object for :class:`.query.Query` objects.""" @@ -42,14 +63,13 @@ class BakedQuery(object): @classmethod def bakery(cls, size=200, _size_alert=None): - """Construct a new bakery.""" + """Construct a new bakery. - _bakery = util.LRUCache(size, size_alert=_size_alert) + :return: an instance of :class:`.Bakery` - def call(initial_fn, *args): - return cls(_bakery, initial_fn, args) + """ - return call + return Bakery(cls, util.LRUCache(size, size_alert=_size_alert)) def _clone(self): b1 = BakedQuery.__new__(BakedQuery) @@ -265,7 +285,7 @@ class Result(object): def __iter__(self): bq = self.bq - if bq._spoiled: + if not self.session.enable_baked_queries or bq._spoiled: return iter(self._as_query()) baked_context = bq._bakery.get(bq._cache_key, None) diff --git a/lib/sqlalchemy/orm/session.py b/lib/sqlalchemy/orm/session.py index 3585166f49..6186ac4f7c 100644 --- a/lib/sqlalchemy/orm/session.py +++ b/lib/sqlalchemy/orm/session.py @@ -588,6 +588,7 @@ class Session(_SessionClassMethods): _enable_transaction_accounting=True, autocommit=False, twophase=False, weak_identity_map=True, binds=None, extension=None, + enable_baked_queries=True, info=None, query_cls=query.Query): r"""Construct a new Session. @@ -661,6 +662,21 @@ class Session(_SessionClassMethods): :class:`.sessionmaker` function, and is not sent directly to the constructor for ``Session``. + :param enable_baked_queries: defaults to ``True``. A flag consumed + by the :mod:`sqlalchemy.ext.baked` extension to determine if + "baked queries" should be cached, as is the normal operation + of this extension. When set to ``False``, all caching is disabled, + including baked queries defined by the calling application as + well as those used internally. Setting this flag to ``False`` + can significantly reduce memory use, however will also degrade + performance for those areas that make use of baked queries + (such as relationship loaders). Additionally, baked query + logic in the calling application or potentially within the ORM + that may be malfunctioning due to cache key collisions or similar + can be flagged by observing if this flag resolves the issue. + + .. versionadded:: 1.2 + :param _enable_transaction_accounting: Defaults to ``True``. A legacy-only flag which when ``False`` disables *all* 0.5-style object accounting on transaction boundaries, including auto-expiry @@ -735,6 +751,7 @@ class Session(_SessionClassMethods): self.autoflush = autoflush self.autocommit = autocommit self.expire_on_commit = expire_on_commit + self.enable_baked_queries = enable_baked_queries self._enable_transaction_accounting = _enable_transaction_accounting self.twophase = twophase self._query_cls = query_cls diff --git a/test/ext/test_baked.py b/test/ext/test_baked.py index 263a1bb6c3..d2fcfbab85 100644 --- a/test/ext/test_baked.py +++ b/test/ext/test_baked.py @@ -447,6 +447,41 @@ class ResultTest(BakedTest): exp ) + def test_disable_on_session(self): + User = self.classes.User + + canary = mock.Mock() + + def fn1(s): + canary.fn1() + return s.query(User.id, User.name).order_by(User.id) + + def fn2(q): + canary.fn2() + return q.filter(User.id == bindparam('id')) + + def fn3(q): + canary.fn3() + return q + + for x in range(3): + bq = self.bakery(fn1) + + bq += fn2 + + sess = Session(autocommit=True, enable_baked_queries=False) + eq_( + bq.add_criteria(fn3)(sess).params(id=7).all(), + [(7, 'jack')] + ) + + eq_( + canary.mock_calls, + [mock.call.fn1(), mock.call.fn2(), mock.call.fn3(), + mock.call.fn1(), mock.call.fn2(), mock.call.fn3(), + mock.call.fn1(), mock.call.fn2(), mock.call.fn3()] + ) + def test_spoiled_full_w_params(self): User = self.classes.User -- 2.39.5