From: Mike Bayer Date: Fri, 11 Oct 2019 18:45:24 +0000 (-0400) Subject: Implement raiseload for deferred columns X-Git-Tag: rel_1_4_0b1~674 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=33119301bb3efa143ebaaef22a7b5170f14a1331;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Implement raiseload for deferred columns Added "raiseload" feature for ORM mapped columns. As part of this change, the behavior of "deferred" is now more strict; an attribute that is set up as "deferred" at the mapper level no longer participates in an "unexpire" operation; that is, when an unexpire loads all the expired columns of an object which are not themselves in a deferred group, those which are mapper-level deferred will never be loaded. Deferral options set at query time should always be reset by an expiration operation. Renames deferred_scalar_loader to expired_attribute_loader Unfortunately we can't have raiseload() do this because it would break existing wildcard behavior. Fixes: #4826 Change-Id: I30d9a30236e0b69134e4094fb7c1ad2267f089d1 --- diff --git a/doc/build/changelog/migration_14.rst b/doc/build/changelog/migration_14.rst index 15b6bcafec..7161867ac1 100644 --- a/doc/build/changelog/migration_14.rst +++ b/doc/build/changelog/migration_14.rst @@ -182,8 +182,80 @@ refined so that it is more compatible with Core. :ticket:`4617` +New Features - ORM +================== + +.. _change_4826: + +Raiseload for Columns +--------------------- + +The "raiseload" feature, which raises :class:`.InvalidRequestError` when an +unloaded attribute is accessed, is now available for column-oriented attributes +using the :paramref:`.orm.defer.raiseload` parameter of :func:`.defer`. This +works in the same manner as that of the :func:`.raiseload` option used by +relationship loading:: + + book = session.query(Book).options(defer(Book.summary, raiseload=True)).first() + + # would raise an exception + book.summary + +To configure column-level raiseload on a mapping, the +:paramref:`.deferred.raiseload` parameter of :func:`.deferred` may be used. The +:func:`.undefer` option may then be used at query time to eagerly load +the attribute:: + + class Book(Base): + __tablename__ = 'book' + + book_id = Column(Integer, primary_key=True) + title = Column(String(200), nullable=False) + summary = deferred(Column(String(2000)), raiseload=True) + excerpt = deferred(Column(Text), raiseload=True) + + book_w_excerpt = session.query(Book).options(undefer(Book.excerpt)).first() + +It was originally considered that the existing :func:`.raiseload` option that +works for :func:`.relationship` attributes be expanded to also support column-oriented +attributes. However, this would break the "wildcard" behavior of :func:`.raiseload`, +which is documented as allowing one to prevent all relationships from loading:: + + session.query(Order).options( + joinedload(Order.items), raiseload('*')) + +Above, if we had expanded :func:`.raiseload` to accommodate for columns as +well, the wildcard would also prevent columns from loading and thus be a +backwards incompatible change; additionally, it's not clear if +:func:`.raiseload` covered both column expressions and relationships, how one +would achieve the effect above of only blocking relationship loads, without +new API being added. So to keep things simple, the option for columns +remains on :func:`.defer`: + + :func:`.raiseload` - query option to raise for relationship loads + + :paramref:`.orm.defer.raiseload` - query option to raise for column expression loads + + +As part of this change, the behavior of "deferred" in conjunction with +attribute expiration has changed. Previously, when an object would be marked +as expired, and then unexpired via the access of one of the expired attributes, +attributes which were mapped as "deferred" at the mapper level would also load. +This has been changed such that an attribute that is deferred in the mapping +will never "unexpire", it only loads when accessed as part of the deferral +loader. + +An attribute that is not mapped as "deferred", however was deferred at query +time via the :func:`.defer` option, will be reset when the object or attribute +is expired; that is, the deferred option is removed. This is the same behavior +as was present previously. + + +.. seealso:: + :ref:`deferred_raiseload` +:ticket:`4826` Behavioral Changes - ORM ======================== diff --git a/doc/build/changelog/unreleased_14/4826.rst b/doc/build/changelog/unreleased_14/4826.rst new file mode 100644 index 0000000000..99535c0b77 --- /dev/null +++ b/doc/build/changelog/unreleased_14/4826.rst @@ -0,0 +1,15 @@ +.. change:: + :tags: feature, orm + :tickets: 4826 + + Added "raiseload" feature for ORM mapped columns via :paramref:`.orm.defer.raiseload` + parameter on :func:`.defer` and :func:`.deferred`. This provides + similar behavior for column-expression mapped attributes as the + :func:`.raiseload` option does for relationship mapped attributes. The + change also includes some behavioral changes to deferred columns regarding + expiration; see the migration notes for details. + + .. seealso:: + + :ref:`change_4826` + diff --git a/doc/build/orm/loading_columns.rst b/doc/build/orm/loading_columns.rst index d128651453..25e64b0ab3 100644 --- a/doc/build/orm/loading_columns.rst +++ b/doc/build/orm/loading_columns.rst @@ -217,6 +217,44 @@ the :class:`.Query` object should raise an informative error message when the above calling style is actually required that describes those cases where explicit use of :class:`.Load` is needed. +.. _deferred_raiseload: + +Raiseload for Deferred Columns +------------------------------ + +.. versionadded:: 1.4 + +The :func:`.deferred` loader option and the corresponding loader strategy also +support the concept of "raiseload", which is a loader strategy that will raise +:class:`.InvalidRequestError` if the attribute is accessed such that it would +need to emit a SQL query in order to be loaded. This behavior is the +column-based equivalent of the :func:`.raiseload` feature for relationship +loading, discussed at :ref:`prevent_lazy_with_raiseload`. Using the +:paramref:`.orm.defer.raiseload` parameter on the :func:`.defer` option, +an exception is raised if the attribute is accessed:: + + book = session.query(Book).options(defer(Book.summary, raiseload=True)).first() + + # would raise an exception + book.summary + +Deferred "raiseload" can be configured at the mapper level via +:paramref:`.orm.deferred.raiseload` on :func:`.deferred`, so that an explicit +:func:`.undefer` is required in order for the attribute to be usable:: + + + class Book(Base): + __tablename__ = 'book' + + book_id = Column(Integer, primary_key=True) + title = Column(String(200), nullable=False) + summary = deferred(Column(String(2000)), raiseload=True) + excerpt = deferred(Column(Text), raiseload=True) + + book_w_excerpt = session.query(Book).options(undefer(Book.excerpt)).first() + + + Column Deferral API ------------------- diff --git a/doc/build/orm/loading_relationships.rst b/doc/build/orm/loading_relationships.rst index cf64833e60..556759eebb 100644 --- a/doc/build/orm/loading_relationships.rst +++ b/doc/build/orm/loading_relationships.rst @@ -292,10 +292,17 @@ Conversely, to set up the raise for just the ``Item`` objects:: session.query(Order).options( joinedload(Order.items).raiseload('*')) + +The :func:`.raiseload` option applies only to relationship attributes. For +column-oriented attributes, the :func:`.defer` option supports the +:paramref:`.orm.defer.raiseload` option which works in the same way. + .. seealso:: :ref:`wildcard_loader_strategies` + :ref:`deferred_raiseload` + .. _joined_eager_loading: Joined Eager Loading diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index 8bd68b4171..e2eb934095 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -154,6 +154,15 @@ def deferred(*columns, **kw): :class:`.Column` object, however a collection is supported in order to support multiple columns mapped under the same attribute. + :param raiseload: boolean, if True, indicates an exception should be raised + if the load operation is to take place. + + .. versionadded:: 1.4 + + .. seealso:: + + :ref:`deferred_raiseload` + :param \**kw: additional keyword arguments passed to :class:`.ColumnProperty`. diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py index 1466f5f47c..83069f113c 100644 --- a/lib/sqlalchemy/orm/attributes.py +++ b/lib/sqlalchemy/orm/attributes.py @@ -468,7 +468,7 @@ class AttributeImpl(object): compare_function=None, active_history=False, parent_token=None, - expire_missing=True, + load_on_unexpire=True, send_modified_events=True, accepts_scalar_loader=None, **kwargs @@ -503,10 +503,13 @@ class AttributeImpl(object): Allows multiple AttributeImpls to all match a single owner attribute. - :param expire_missing: - if False, don't add an "expiry" callable to this attribute - during state.expire_attributes(None), if no value is present - for this key. + :param load_on_unexpire: + if False, don't include this attribute in a load-on-expired + operation, i.e. the "expired_attribute_loader" process. + The attribute can still be in the "expired" list and be + considered to be "expired". Previously, this flag was called + "expire_missing" and is only used by a deferred column + attribute. :param send_modified_events: if False, the InstanceState._modified_event method will have no @@ -534,7 +537,7 @@ class AttributeImpl(object): if active_history: self.dispatch._active_history = True - self.expire_missing = expire_missing + self.load_on_unexpire = load_on_unexpire self._modified_token = Event(self, OP_MODIFIED) __slots__ = ( @@ -546,7 +549,7 @@ class AttributeImpl(object): "parent_token", "send_modified_events", "is_equal", - "expire_missing", + "load_on_unexpire", "_modified_token", "accepts_scalar_loader", ) @@ -683,6 +686,7 @@ class AttributeImpl(object): if ( self.accepts_scalar_loader + and self.load_on_unexpire and key in state.expired_attributes ): value = state._load_expired(state, passive) diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py index e52b6d8bb5..6f8d192934 100644 --- a/lib/sqlalchemy/orm/base.py +++ b/lib/sqlalchemy/orm/base.py @@ -206,6 +206,8 @@ _SET_DEFERRED_EXPIRED = util.symbol("SET_DEFERRED_EXPIRED") _DEFER_FOR_STATE = util.symbol("DEFER_FOR_STATE") +_RAISE_FOR_STATE = util.symbol("RAISE_FOR_STATE") + def _assertions(*assertions): @util.decorator diff --git a/lib/sqlalchemy/orm/descriptor_props.py b/lib/sqlalchemy/orm/descriptor_props.py index dd482bf069..3be5502cef 100644 --- a/lib/sqlalchemy/orm/descriptor_props.py +++ b/lib/sqlalchemy/orm/descriptor_props.py @@ -36,7 +36,7 @@ class DescriptorProperty(MapperProperty): class _ProxyImpl(object): accepts_scalar_loader = False - expire_missing = True + load_on_unexpire = True collection = False @property diff --git a/lib/sqlalchemy/orm/instrumentation.py b/lib/sqlalchemy/orm/instrumentation.py index 61184ee0a0..ecb8d7857f 100644 --- a/lib/sqlalchemy/orm/instrumentation.py +++ b/lib/sqlalchemy/orm/instrumentation.py @@ -49,12 +49,31 @@ class ClassManager(dict): _state_setter = staticmethod(util.attrsetter(STATE_ATTR)) - deferred_scalar_loader = None + expired_attribute_loader = None + "previously known as deferred_scalar_loader" original_init = object.__init__ factory = None + @property + @util.deprecated( + "1.4", + message="The ClassManager.deferred_scalar_loader attribute is now " + "named expired_attribute_loader", + ) + def deferred_scalar_loader(self): + return self.expired_attribute_loader + + @deferred_scalar_loader.setter + @util.deprecated( + "1.4", + message="The ClassManager.deferred_scalar_loader attribute is now " + "named expired_attribute_loader", + ) + def deferred_scalar_loader(self, obj): + self.expired_attribute_loader = obj + def __init__(self, class_): self.class_ = class_ self.info = {} diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 09d1858b94..abb6b14cc1 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -538,9 +538,10 @@ class StrategizedProperty(MapperProperty): return self._strategies[key] except KeyError: cls = self._strategy_lookup(self, *key) - self._strategies[key] = self._strategies[cls] = strategy = cls( - self, key - ) + # this previosuly was setting self._strategies[cls], that's + # a bad idea; should use strategy key at all times because every + # strategy has multiple keys at this point + self._strategies[key] = strategy = cls(self, key) return strategy def setup(self, context, query_entity, path, adapter, **kwargs): diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py index 106ea7985e..8de7d5a8b8 100644 --- a/lib/sqlalchemy/orm/loading.py +++ b/lib/sqlalchemy/orm/loading.py @@ -21,6 +21,7 @@ from . import exc as orm_exc from . import path_registry from . import strategy_options from .base import _DEFER_FOR_STATE +from .base import _RAISE_FOR_STATE from .base import _SET_DEFERRED_EXPIRED from .util import _none_set from .util import aliased @@ -384,6 +385,8 @@ def _instance_processor( # searching in the result to see if the column might # be present in some unexpected way. populators["expire"].append((prop.key, False)) + elif col is _RAISE_FOR_STATE: + populators["new"].append((prop.key, prop._raise_column_loader)) else: getter = None # the "adapter" can be here via different paths, diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 0d8d454eb4..376ad19233 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1247,7 +1247,7 @@ class Mapper(InspectionAttr): self.class_manager = manager manager.mapper = self - manager.deferred_scalar_loader = util.partial( + manager.expired_attribute_loader = util.partial( loading.load_scalar_attributes, self ) diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index f8bf069262..2e6e105fa2 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -53,6 +53,8 @@ class ColumnProperty(StrategizedProperty): "_is_polymorphic_discriminator", "_mapped_by_synonym", "_deferred_column_loader", + "_raise_column_loader", + "raiseload", ) def __init__(self, *columns, **kwargs): @@ -115,6 +117,16 @@ class ColumnProperty(StrategizedProperty): :param info: Optional data dictionary which will be populated into the :attr:`.MapperProperty.info` attribute of this object. + :param raiseload: if True, indicates the column should raise an error + when undeferred, rather than loading the value. This can be + altered at query time by using the :func:`.deferred` option with + raiseload=False. + + .. versionadded:: 1.4 + + .. seealso:: + + :ref:`deferred_raiseload` """ super(ColumnProperty, self).__init__() @@ -129,6 +141,7 @@ class ColumnProperty(StrategizedProperty): ] self.group = kwargs.pop("group", None) self.deferred = kwargs.pop("deferred", False) + self.raiseload = kwargs.pop("raiseload", False) self.instrument = kwargs.pop("_instrument", True) self.comparator_factory = kwargs.pop( "comparator_factory", self.__class__.Comparator @@ -163,6 +176,8 @@ class ColumnProperty(StrategizedProperty): ("deferred", self.deferred), ("instrument", self.instrument), ) + if self.raiseload: + self.strategy_key += (("raiseload", True),) @util.dependencies("sqlalchemy.orm.state", "sqlalchemy.orm.strategies") def _memoized_attr__deferred_column_loader(self, state, strategies): @@ -172,6 +187,14 @@ class ColumnProperty(StrategizedProperty): self.key, ) + @util.dependencies("sqlalchemy.orm.state", "sqlalchemy.orm.strategies") + def _memoized_attr__raise_column_loader(self, state, strategies): + return state.InstanceState._instance_level_callable_processor( + self.parent.class_manager, + strategies.LoadDeferredColumns(self.key, True), + self.key, + ) + def __clause_element__(self): """Allow the ColumnProperty to work in expression before it is turned into an instrumented attribute. diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py index ead9bf2bbe..c57af0784f 100644 --- a/lib/sqlalchemy/orm/state.py +++ b/lib/sqlalchemy/orm/state.py @@ -593,11 +593,7 @@ class InstanceState(interfaces.InspectionAttrInfo): del self.__dict__["parents"] self.expired_attributes.update( - [ - impl.key - for impl in self.manager._loader_impls - if impl.expire_missing or impl.key in dict_ - ] + [impl.key for impl in self.manager._loader_impls] ) if self.callables: @@ -671,8 +667,13 @@ class InstanceState(interfaces.InspectionAttrInfo): return PASSIVE_NO_RESULT toload = self.expired_attributes.intersection(self.unmodified) + toload = toload.difference( + attr + for attr in toload + if not self.manager[attr].impl.load_on_unexpire + ) - self.manager.deferred_scalar_loader(self, toload) + self.manager.expired_attribute_loader(self, toload) # if the loader failed, or this # instance state didn't have an identity, @@ -719,11 +720,7 @@ class InstanceState(interfaces.InspectionAttrInfo): was never populated or modified. """ - return self.unloaded.intersection( - attr - for attr in self.manager - if self.manager[attr].impl.expire_missing - ) + return self.unloaded @property def _unloaded_non_object(self): diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index f82fc2c576..59877a521a 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -21,6 +21,7 @@ from . import query from . import unitofwork from . import util as orm_util from .base import _DEFER_FOR_STATE +from .base import _RAISE_FOR_STATE from .base import _SET_DEFERRED_EXPIRED from .interfaces import LoaderStrategy from .interfaces import StrategizedProperty @@ -287,11 +288,14 @@ class ExpressionColumnLoader(ColumnLoader): @log.class_logger @properties.ColumnProperty.strategy_for(deferred=True, instrument=True) +@properties.ColumnProperty.strategy_for( + deferred=True, instrument=True, raiseload=True +) @properties.ColumnProperty.strategy_for(do_nothing=True) class DeferredColumnLoader(LoaderStrategy): """Provide loading behavior for a deferred :class:`.ColumnProperty`.""" - __slots__ = "columns", "group" + __slots__ = "columns", "group", "raiseload" def __init__(self, parent, strategy_key): super(DeferredColumnLoader, self).__init__(parent, strategy_key) @@ -299,6 +303,7 @@ class DeferredColumnLoader(LoaderStrategy): raise NotImplementedError( "Deferred loading for composite " "types not implemented yet" ) + self.raiseload = self.strategy_opts.get("raiseload", False) self.columns = self.parent_property.columns self.group = self.parent_property.group @@ -306,14 +311,23 @@ class DeferredColumnLoader(LoaderStrategy): self, context, path, loadopt, mapper, result, adapter, populators ): - # this path currently does not check the result - # for the column; this is because in most cases we are - # working just with the setup_query() directive which does - # not support this, and the behavior here should be consistent. + # for a DeferredColumnLoader, this method is only used during a + # "row processor only" query; see test_deferred.py -> + # tests with "rowproc_only" in their name. As of the 1.0 series, + # loading._instance_processor doesn't use a "row processing" function + # to populate columns, instead it uses data in the "populators" + # dictionary. Normally, the DeferredColumnLoader.setup_query() + # sets up that data in the "memoized_populators" dictionary + # and "create_row_processor()" here is never invoked. if not self.is_class_level: - set_deferred_for_local_state = ( - self.parent_property._deferred_column_loader - ) + if self.raiseload: + set_deferred_for_local_state = ( + self.parent_property._raise_column_loader + ) + else: + set_deferred_for_local_state = ( + self.parent_property._deferred_column_loader + ) populators["new"].append((self.key, set_deferred_for_local_state)) else: populators["expire"].append((self.key, False)) @@ -327,7 +341,7 @@ class DeferredColumnLoader(LoaderStrategy): useobject=False, compare_function=self.columns[0].type.compare_values, callable_=self._load_for_state, - expire_missing=False, + load_on_unexpire=False, ) def setup_query( @@ -374,8 +388,10 @@ class DeferredColumnLoader(LoaderStrategy): ) elif self.is_class_level: memoized_populators[self.parent_property] = _SET_DEFERRED_EXPIRED - else: + elif not self.raiseload: memoized_populators[self.parent_property] = _DEFER_FOR_STATE + else: + memoized_populators[self.parent_property] = _RAISE_FOR_STATE def _load_for_state(self, state, passive): if not state.key: @@ -408,6 +424,9 @@ class DeferredColumnLoader(LoaderStrategy): % (orm_util.state_str(state), self.key) ) + if self.raiseload: + self._invoke_raise_load(state, passive, "raise") + query = session.query(localparent) if ( loading.load_on_ident( @@ -419,19 +438,33 @@ class DeferredColumnLoader(LoaderStrategy): return attributes.ATTR_WAS_SET + def _invoke_raise_load(self, state, passive, lazy): + raise sa_exc.InvalidRequestError( + "'%s' is not available due to raiseload=True" % (self,) + ) + class LoadDeferredColumns(object): """serializable loader object used by DeferredColumnLoader""" - def __init__(self, key): + def __init__(self, key, raiseload=False): self.key = key + self.raiseload = raiseload def __call__(self, state, passive=attributes.PASSIVE_OFF): key = self.key localparent = state.manager.mapper prop = localparent._props[key] - strategy = prop._strategies[DeferredColumnLoader] + if self.raiseload: + strategy_key = ( + ("deferred", True), + ("instrument", True), + ("raiseload", True), + ) + else: + strategy_key = (("deferred", True), ("instrument", True)) + strategy = prop._get_strategy(strategy_key) return strategy._load_for_state(state, passive) diff --git a/lib/sqlalchemy/orm/strategy_options.py b/lib/sqlalchemy/orm/strategy_options.py index 00c91dab59..26f47f6169 100644 --- a/lib/sqlalchemy/orm/strategy_options.py +++ b/lib/sqlalchemy/orm/strategy_options.py @@ -1357,7 +1357,7 @@ def noload(*keys): @loader_option() def raiseload(loadopt, attr, sql_only=False): - """Indicate that the given relationship attribute should disallow lazy loads. + """Indicate that the given attribute should raise an error if accessed. A relationship attribute configured with :func:`.orm.raiseload` will raise an :exc:`~sqlalchemy.exc.InvalidRequestError` upon access. The @@ -1367,15 +1367,19 @@ def raiseload(loadopt, attr, sql_only=False): to read through SQL logs to ensure lazy loads aren't occurring, this strategy will cause them to raise immediately. - :param sql_only: if True, raise only if the lazy load would emit SQL, - but not if it is only checking the identity map, or determining that - the related value should just be None due to missing keys. When False, - the strategy will raise for all varieties of lazyload. + :func:`.orm.raiseload` applies to :func:`.relationship` attributes only. + In order to apply raise-on-SQL behavior to a column-based attribute, + use the :paramref:`.orm.defer.raiseload` parameter on the :func:`.defer` + loader option. + + :param sql_only: if True, raise only if the lazy load would emit SQL, but + not if it is only checking the identity map, or determining that the + related value should just be None due to missing keys. When False, the + strategy will raise for all varieties of relationship loading. This function is part of the :class:`.Load` interface and supports both method-chained and standalone operation. - :func:`.orm.raiseload` applies to :func:`.relationship` attributes only. .. versionadded:: 1.1 @@ -1385,6 +1389,8 @@ def raiseload(loadopt, attr, sql_only=False): :ref:`prevent_lazy_with_raiseload` + :ref:`deferred_raiseload` + """ return loadopt.set_relationship_strategy( @@ -1440,7 +1446,7 @@ def defaultload(*keys): @loader_option() -def defer(loadopt, key): +def defer(loadopt, key, raiseload=False): r"""Indicate that the given column-oriented attribute should be deferred, e.g. not loaded until accessed. @@ -1480,6 +1486,16 @@ def defer(loadopt, key): :param key: Attribute to be deferred. + :param raiseload: raise :class:`.InvalidRequestError` if the column + value is to be loaded from emitting SQL. Used to prevent unwanted + SQL from being emitted. + + .. versionadded:: 1.4 + + .. seealso:: + + :ref:`deferred_raiseload` + :param \*addl_attrs: This option supports the old 0.8 style of specifying a path as a series of attributes, which is now superseded by the method-chained style. @@ -1497,13 +1513,14 @@ def defer(loadopt, key): :func:`.orm.undefer` """ - return loadopt.set_column_strategy( - (key,), {"deferred": True, "instrument": True} - ) + strategy = {"deferred": True, "instrument": True} + if raiseload: + strategy["raiseload"] = True + return loadopt.set_column_strategy((key,), strategy) @defer._add_unbound_fn -def defer(key, *addl_attrs): +def defer(key, *addl_attrs, **kw): if addl_attrs: util.warn_deprecated( "The *addl_attrs on orm.defer is deprecated. Please use " @@ -1511,7 +1528,7 @@ def defer(key, *addl_attrs): "indicate a path." ) return _UnboundLoad._from_keys( - _UnboundLoad.defer, (key,) + addl_attrs, False, {} + _UnboundLoad.defer, (key,) + addl_attrs, False, kw ) diff --git a/test/ext/test_extendedattr.py b/test/ext/test_extendedattr.py index 161c187625..46df83a301 100644 --- a/test/ext/test_extendedattr.py +++ b/test/ext/test_extendedattr.py @@ -229,7 +229,7 @@ class UserDefinedExtensionTest(_ExtBase, fixtures.ORMTest): return attributes.ATTR_WAS_SET manager = register_class(Foo) - manager.deferred_scalar_loader = loader + manager.expired_attribute_loader = loader attributes.register_attribute( Foo, "a", uselist=False, useobject=False ) diff --git a/test/orm/test_attributes.py b/test/orm/test_attributes.py index 9f7979116a..347dd4e46d 100644 --- a/test/orm/test_attributes.py +++ b/test/orm/test_attributes.py @@ -449,7 +449,7 @@ class AttributesTest(fixtures.ORMTest): instrumentation.register_class(Foo) manager = attributes.manager_of_class(Foo) - manager.deferred_scalar_loader = loader + manager.expired_attribute_loader = loader attributes.register_attribute(Foo, "a", uselist=False, useobject=False) attributes.register_attribute(Foo, "b", uselist=False, useobject=False) @@ -495,7 +495,7 @@ class AttributesTest(fixtures.ORMTest): instrumentation.register_class(MyTest) manager = attributes.manager_of_class(MyTest) - manager.deferred_scalar_loader = loader + manager.expired_attribute_loader = loader attributes.register_attribute( MyTest, "a", uselist=False, useobject=False ) @@ -2245,7 +2245,7 @@ class HistoryTest(fixtures.TestBase): def scalar_loader(state, toload): state.dict["someattr"] = "one" - state.manager.deferred_scalar_loader = scalar_loader + state.manager.expired_attribute_loader = scalar_loader eq_(self._someattr_history(f), ((), ["one"], ())) diff --git a/test/orm/test_deferred.py b/test/orm/test_deferred.py index 89e58061dd..4f76fdc039 100644 --- a/test/orm/test_deferred.py +++ b/test/orm/test_deferred.py @@ -874,6 +874,72 @@ class DeferredOptionsTest(AssertsCompiledSQL, _fixtures.FixtureTest): # self.sql_count_(0, go) self.sql_count_(1, go) + def test_raise_on_col_rowproc_only(self): + orders, Order = self.tables.orders, self.classes.Order + + mapper( + Order, + orders, + properties={ + "description": deferred(orders.c.description, raiseload=True) + }, + ) + + sess = create_session() + stmt = sa.select([Order]).order_by(Order.id) + o1 = (sess.query(Order).from_statement(stmt).all())[0] + + assert_raises_message( + sa.exc.InvalidRequestError, + "'Order.description' is not available due to raiseload=True", + getattr, + o1, + "description", + ) + + def test_locates_col_w_option_rowproc_only(self): + orders, Order = self.tables.orders, self.classes.Order + + mapper(Order, orders) + + sess = create_session() + stmt = sa.select([Order]).order_by(Order.id) + o1 = ( + sess.query(Order) + .from_statement(stmt) + .options(defer(Order.description)) + .all() + )[0] + + def go(): + eq_(o1.description, "order 1") + + # prior to 1.0 we'd search in the result for this column + # self.sql_count_(0, go) + self.sql_count_(1, go) + + def test_raise_on_col_w_option_rowproc_only(self): + orders, Order = self.tables.orders, self.classes.Order + + mapper(Order, orders) + + sess = create_session() + stmt = sa.select([Order]).order_by(Order.id) + o1 = ( + sess.query(Order) + .from_statement(stmt) + .options(defer(Order.description, raiseload=True)) + .all() + )[0] + + assert_raises_message( + sa.exc.InvalidRequestError, + "'Order.description' is not available due to raiseload=True", + getattr, + o1, + "description", + ) + def test_deep_options(self): users, items, order_items, Order, Item, User, orders = ( self.tables.users, @@ -1781,3 +1847,177 @@ class WithExpressionTest(fixtures.DeclarativeMappedTest): ) q.first() eq_(a1.my_expr, 5) + + +class RaiseLoadTest(fixtures.DeclarativeMappedTest): + @classmethod + def setup_classes(cls): + Base = cls.DeclarativeBasic + + class A(fixtures.ComparableEntity, Base): + __tablename__ = "a" + id = Column(Integer, primary_key=True) + x = Column(Integer) + y = deferred(Column(Integer)) + z = deferred(Column(Integer), raiseload=True) + + @classmethod + def insert_data(cls): + A = cls.classes.A + s = Session() + s.add(A(id=1, x=2, y=3, z=4)) + s.commit() + + def test_mapper_raise(self): + A = self.classes.A + s = Session() + a1 = s.query(A).first() + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.z' is not available due to raiseload=True", + getattr, + a1, + "z", + ) + eq_(a1.x, 2) + + def test_mapper_defer_unraise(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.z)).first() + assert "z" not in a1.__dict__ + eq_(a1.z, 4) + eq_(a1.x, 2) + + def test_mapper_undefer_unraise(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(undefer(A.z)).first() + assert "z" in a1.__dict__ + eq_(a1.z, 4) + eq_(a1.x, 2) + + def test_deferred_raise_option_raise_column_plain(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.x)).first() + a1.x + + s.close() + + a1 = s.query(A).options(defer(A.x, raiseload=True)).first() + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.x' is not available due to raiseload=True", + getattr, + a1, + "x", + ) + + def test_deferred_raise_option_load_column_unexpire(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.x, raiseload=True)).first() + s.expire(a1, ["x"]) + + # after expire(), options are cleared. relationship w/ raiseload + # works this way also + eq_(a1.x, 2) + + def test_mapper_raise_after_expire_attr(self): + A = self.classes.A + s = Session() + a1 = s.query(A).first() + + s.expire(a1, ["z"]) + + # raises even after expire() + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.z' is not available due to raiseload=True", + getattr, + a1, + "z", + ) + + def test_mapper_raise_after_expire_obj(self): + A = self.classes.A + s = Session() + a1 = s.query(A).first() + + s.expire(a1) + + # raises even after expire() + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.z' is not available due to raiseload=True", + getattr, + a1, + "z", + ) + + def test_mapper_raise_after_modify_attr_expire_obj(self): + A = self.classes.A + s = Session() + a1 = s.query(A).first() + + a1.z = 10 + s.expire(a1) + + # raises even after expire() + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.z' is not available due to raiseload=True", + getattr, + a1, + "z", + ) + + def test_deferred_raise_option_load_after_expire_obj(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.y, raiseload=True)).first() + + s.expire(a1) + + # after expire(), options are cleared. relationship w/ raiseload + # works this way also + eq_(a1.y, 3) + + def test_option_raiseload_unexpire_modified_obj(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.y, raiseload=True)).first() + + a1.y = 10 + s.expire(a1) + + # after expire(), options are cleared. relationship w/ raiseload + # works this way also + eq_(a1.y, 3) + + def test_option_raise_deferred(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.y, raiseload=True)).first() + + assert_raises_message( + sa.exc.InvalidRequestError, + "'A.y' is not available due to raiseload=True", + getattr, + a1, + "y", + ) + + def test_does_expire_cancel_normal_defer_option(self): + A = self.classes.A + s = Session() + a1 = s.query(A).options(defer(A.x)).first() + + # expire object + s.expire(a1) + + # unexpire object + eq_(a1.id, 1) + + assert "x" in a1.__dict__ diff --git a/test/orm/test_deprecations.py b/test/orm/test_deprecations.py index 8fc3c0f0bb..9c7ae19673 100644 --- a/test/orm/test_deprecations.py +++ b/test/orm/test_deprecations.py @@ -870,6 +870,30 @@ class DeprecatedInhTest(_poly_fixtures._Polymorphic): class DeprecatedMapperTest(_fixtures.FixtureTest, AssertsCompiledSQL): __dialect__ = "default" + def test_deferred_scalar_loader_name_change(self): + class Foo(object): + pass + + def myloader(*arg, **kw): + pass + + instrumentation.register_class(Foo) + manager = instrumentation.manager_of_class(Foo) + + with testing.expect_deprecated( + "The ClassManager.deferred_scalar_loader attribute is now named " + "expired_attribute_loader" + ): + manager.deferred_scalar_loader = myloader + + is_(manager.expired_attribute_loader, myloader) + + with testing.expect_deprecated( + "The ClassManager.deferred_scalar_loader attribute is now named " + "expired_attribute_loader" + ): + is_(manager.deferred_scalar_loader, myloader) + def test_polymorphic_union_w_select(self): users, addresses = self.tables.users, self.tables.addresses diff --git a/test/orm/test_expire.py b/test/orm/test_expire.py index 6c61e9c5d1..083c2f4651 100644 --- a/test/orm/test_expire.py +++ b/test/orm/test_expire.py @@ -4,6 +4,7 @@ import sqlalchemy as sa from sqlalchemy import exc as sa_exc from sqlalchemy import FetchedValue from sqlalchemy import ForeignKey +from sqlalchemy import inspect from sqlalchemy import Integer from sqlalchemy import String from sqlalchemy import testing @@ -191,8 +192,23 @@ class ExpireTest(_fixtures.FixtureTest): o1 = s.query(Order).first() assert "description" not in o1.__dict__ s.expire(o1) + + # the deferred attribute is listed as expired (new in 1.4) + eq_( + inspect(o1).expired_attributes, + {"id", "isopen", "address_id", "user_id", "description"}, + ) + + # unexpire by accessing isopen assert o1.isopen is not None + + # all expired_attributes are cleared + eq_(inspect(o1).expired_attributes, set()) + + # but description wasn't loaded (new in 1.4) assert "description" not in o1.__dict__ + + # loads using deferred callable assert o1.description def test_deferred_notfound(self): @@ -992,25 +1008,26 @@ class ExpireTest(_fixtures.FixtureTest): assert "isopen" not in o.__dict__ assert "description" not in o.__dict__ - # test that expired attribute access refreshes + # test that expired attribute access does not refresh # the deferred def go(): assert o.isopen == 1 assert o.description == "order 3" - self.assert_sql_count(testing.db, go, 1) + # requires two statements + self.assert_sql_count(testing.db, go, 2) sess.expire(o, ["description", "isopen"]) assert "isopen" not in o.__dict__ assert "description" not in o.__dict__ - # test that the deferred attribute triggers the full + # test that the deferred attribute does not trigger the full # reload def go(): assert o.description == "order 3" assert o.isopen == 1 - self.assert_sql_count(testing.db, go, 1) + self.assert_sql_count(testing.db, go, 2) sa.orm.clear_mappers() @@ -1195,13 +1212,15 @@ class ExpireTest(_fixtures.FixtureTest): u1 = sess.query(User).options(undefer(User.name)).first() del u1.name sess.expire(u1) - assert "name" not in attributes.instance_state(u1).expired_attributes + assert "name" in attributes.instance_state(u1).expired_attributes assert "name" not in attributes.instance_state(u1).callables # single attribute expire, the attribute gets the callable sess.expunge_all() u1 = sess.query(User).options(undefer(User.name)).first() sess.expire(u1, ["name"]) + + # the expire cancels the undefer assert "name" in attributes.instance_state(u1).expired_attributes assert "name" not in attributes.instance_state(u1).callables @@ -1305,7 +1324,7 @@ class ExpireTest(_fixtures.FixtureTest): item = s.query(Order).first() s.expire(item, ["isopen", "description"]) item.isopen - assert "description" in item.__dict__ + assert "description" not in item.__dict__ class PolymorphicExpireTest(fixtures.MappedTest): diff --git a/test/orm/test_mapper.py b/test/orm/test_mapper.py index 2622ea640b..29b64547ba 100644 --- a/test/orm/test_mapper.py +++ b/test/orm/test_mapper.py @@ -3395,6 +3395,87 @@ class RaiseLoadTest(_fixtures.FixtureTest): lambda: a1.user, ) + def test_raiseload_wildcard_all_classes_option(self): + Address, addresses, users, User = ( + self.classes.Address, + self.tables.addresses, + self.tables.users, + self.classes.User, + ) + + mapper(Address, addresses) + mapper( + User, + users, + properties=dict(addresses=relationship(Address, backref="user")), + ) + q = ( + create_session() + .query(User, Address) + .join(Address, User.id == Address.user_id) + ) + + u1, a1 = q.options(sa.orm.raiseload("*")).filter(User.id == 7).first() + + assert_raises_message( + sa.exc.InvalidRequestError, + "'User.addresses' is not available due to lazy='raise'", + lambda: u1.addresses, + ) + + assert_raises_message( + sa.exc.InvalidRequestError, + "'Address.user' is not available due to lazy='raise'", + lambda: a1.user, + ) + + # columns still work + eq_(u1.id, 7) + eq_(a1.id, 1) + + def test_raiseload_wildcard_specific_class_option(self): + Address, addresses, users, User = ( + self.classes.Address, + self.tables.addresses, + self.tables.users, + self.classes.User, + ) + + mapper(Address, addresses) + mapper( + User, + users, + properties=dict(addresses=relationship(Address, backref="user")), + ) + q = ( + create_session() + .query(User, Address) + .join(Address, User.id == Address.user_id) + ) + + u1, a1 = ( + q.options(sa.orm.Load(Address).raiseload("*")) + .filter(User.id == 7) + .first() + ) + + # User doesn't raise + def go(): + eq_(u1.addresses, [a1]) + + self.assert_sql_count(testing.db, go, 1) + + # Address does + assert_raises_message( + sa.exc.InvalidRequestError, + "'Address.user' is not available due to lazy='raise'", + lambda: a1.user, + ) + + # columns still work + eq_(u1.id, 7) + eq_(a1.id, 1) + class RequirementsTest(fixtures.MappedTest):