This can occur when user defined code inadvertently
triggers flushes on not-fully-loaded objects.
+ - lazy loads for relationship attributes now use
+ the current state, not the "committed" state,
+ of foreign and primary key attributes
+ when issuing SQL, if a flush is not in process.
+ Previously, only the database-committed state would
+ be used. In particular, this would cause a many-to-one
+ get()-on-lazyload operation to fail, as autoflush
+ is not triggered on these loads when the attributes are
+ determined and the "committed" state may not be
+ available. [ticket:1910]
+
+ - A new flag on relationship(), load_on_pending, allows
+ the lazy loader to fire off on pending objects without a
+ flush taking place, as well as a transient object that's
+ been manually "attached" to the session. Note that this
+ flag blocks attribute events from taking place when an
+ object is loaded, so backrefs aren't available until
+ after a flush. The flag is only intended for very
+ specific use cases.
+
- Slight improvement to the behavior of "passive_updates=False"
when placed only on the many-to-one side of a
relationship; documentation has been clarified
raise ValueError("Couldn't find suitable create_engine call to replace '%s' in it" % oldfile)
return s
-for filename in ('ormtutorial', 'sqlexpression'):
+for filename in ('orm/tutorial', 'core/tutorial'):
filename = '%s.rst' % filename
s = open(filename).read()
#s = replace_file(s, ':memory:')
* None - a synonym for 'noload'
Detailed discussion of loader strategies is at :ref:`loading_toplevel`.
+
+ :param load_on_pending:
+ Indicates loading behavior for transient or pending parent objects.
+
+ This is an advanced user feature that will cause the lazy-loader to
+ issue a query for a parent object that is not persistent, meaning it has
+ never been flushed. This may take effect for a pending object when
+ autoflush is disabled, or for a transient object that has been
+ "attached" to a :class:`.Session` but is not part of its pending
+ collection. Attachment of transient objects to the session without
+ moving to the "pending" state is not a supported behavior at this time.
+
+ Note that the load of related objects on a pending or transient object
+ also does not trigger any attribute change events - no user-defined
+ events will be emitted for these attributes, and if and when the
+ object is ultimately flushed, only the user-specific foreign key
+ attributes will be part of the modified state.
:param order_by:
indicates the ordering that should be applied when loading these
# this is used by backrefs.
PASSIVE_NO_FETCH = util.symbol('PASSIVE_NO_FETCH')
-"""Symbol indicating that loader callables should not boe fired off.
+"""Symbol indicating that loader callables should not be fired off.
Non-initialized attributes should be initialized to an empty value."""
+PASSIVE_ONLY_PERSISTENT = util.symbol('PASSIVE_ONLY_PERSISTENT')
+"""Symbol indicating that loader callables should only fire off for
+persistent objects.
+
+Loads of "previous" values during change events use this flag.
+"""
+
PASSIVE_OFF = False #util.symbol('PASSIVE_OFF')
"""Symbol indicating that loader callables should be executed."""
return
if self.active_history:
- old = self.get(state, dict_)
+ old = self.get(state, dict_, passive=PASSIVE_ONLY_PERSISTENT)
else:
old = self.get(state, dict_, passive=PASSIVE_NO_FETCH)
-
+
value = self.fire_replace_event(state, dict_, value, old, initiator)
dict_[self.key] = value
else:
new_values = list(iterable)
- old = self.get(state, dict_)
-
- # ignore re-assignment of the current collection, as happens
- # implicitly with in-place operators (foo.collection |= other)
- if old is iterable:
+ old = self.get(state, dict_, passive=PASSIVE_ONLY_PERSISTENT)
+ if old is PASSIVE_NO_RESULT:
+ old = self.initialize(state, dict_)
+ elif old is iterable:
+ # ignore re-assignment of the current collection, as happens
+ # implicitly with in-place operators (foo.collection |= other)
return
state.modified_event(dict_, self, True, old)
-
+
old_collection = self.get_collection(state, dict_, old)
dict_[self.key] = user_data
def set(self, state, child, oldchild, initiator):
if oldchild is child:
return child
-
+
if oldchild is not None and oldchild is not PASSIVE_NO_RESULT:
# With lazy=None, there's no guarantee that the full collection is
# present when updating via a backref.
def append_with_event(self, item, initiator=None):
"""Add an entity to the collection, firing mutation events."""
+
getattr(self._data(), '_sa_appender')(item, _sa_initiator=initiator)
def append_without_event(self, item):
column in self.primary_key]
# TODO: improve names?
- def _get_state_attr_by_column(self, state, dict_, column):
- return self._columntoproperty[column]._getattr(state, dict_, column)
+ def _get_state_attr_by_column(self, state, dict_, column, passive=False):
+ return self._columntoproperty[column]._getattr(state, dict_, column, passive=passive)
def _set_state_attr_by_column(self, state, dict_, column, value):
return self._columntoproperty[column]._setattr(state, dict_, value, column)
group=self.group,
*self.columns)
- def _getattr(self, state, dict_, column):
- return state.get_impl(self.key).get(state, dict_)
+ def _getattr(self, state, dict_, column, passive=False):
+ return state.get_impl(self.key).get(state, dict_, passive=passive)
def _getcommitted(self, state, dict_, column, passive=False):
return state.get_impl(self.key).\
# which issues assertions that do not apply to CompositeColumnProperty
super(ColumnProperty, self).do_init()
- def _getattr(self, state, dict_, column):
- obj = state.get_impl(self.key).get(state, dict_)
+ def _getattr(self, state, dict_, column, passive=False):
+ obj = state.get_impl(self.key).get(state, dict_, passive=passive)
return self.get_col_value(column, obj)
def _getcommitted(self, state, dict_, column, passive=False):
comparator_factory=None,
single_parent=False, innerjoin=False,
doc=None,
+ load_on_pending=False,
strategy_class=None, _local_remote_pairs=None, query_class=None):
self.uselist = uselist
self.join_depth = join_depth
self.local_remote_pairs = _local_remote_pairs
self.extension = extension
+ self.load_on_pending = load_on_pending
self.comparator_factory = comparator_factory or \
RelationshipProperty.Comparator
self.comparator = self.comparator_factory(self, None)
def compare(self, op, value,
value_is_parent=False,
- alias_secondary=True,
- detect_transient_pending=False):
+ alias_secondary=True):
if op == operators.eq:
if value is None:
if self.uselist:
else:
return self._optimized_compare(None,
value_is_parent=value_is_parent,
- detect_transient_pending=detect_transient_pending,
alias_secondary=alias_secondary)
else:
return self._optimized_compare(value,
value_is_parent=value_is_parent,
- detect_transient_pending=detect_transient_pending,
alias_secondary=alias_secondary)
else:
return op(self.comparator, value)
def _optimized_compare(self, value, value_is_parent=False,
adapt_source=None,
- detect_transient_pending=False,
alias_secondary=True):
if value is not None:
value = attributes.instance_state(value)
return self._get_strategy(strategies.LazyLoader).lazy_clause(value,
reverse_direction=not value_is_parent,
alias_secondary=alias_secondary,
- detect_transient_pending=detect_transient_pending,
adapt_source=adapt_source)
def __str__(self):
previous = dict_[attr.key]
else:
previous = attr.get(self, dict_)
-
+
if should_copy and previous not in (None, NO_VALUE, NEVER_SET):
previous = attr.copy(previous)
def __init__(self, state, key):
self.state, self.key = state, key
- def __call__(self, **kw):
- if kw.get('passive') is attributes.PASSIVE_NO_FETCH:
+ def __call__(self, passive=False):
+ if passive is attributes.PASSIVE_NO_FETCH:
return attributes.PASSIVE_NO_RESULT
state = self.state
def lazy_clause(self, state, reverse_direction=False,
alias_secondary=False,
- adapt_source=None,
- detect_transient_pending=False):
+ adapt_source=None):
if state is None:
return self._lazy_none_clause(
reverse_direction,
o = state.obj() # strong ref
dict_ = attributes.instance_dict(o)
- def visit_bindparam(bindparam):
- if bindparam.key in bind_to_col:
- # using a flag to enable "detect transient pending" so that
- # the slightly different usage paradigm of "dynamic" loaders
- # continue to work as expected, i.e. that all pending objects
- # should use the "post flush" attributes, and to limit this
- # newer behavior to the query.with_parent() method.
- # It would be nice to do away with this flag.
-
- if detect_transient_pending and \
- (not state.key or not state.session_id):
- bindparam.value = mapper._get_state_attr_by_column(
- state, dict_, bind_to_col[bindparam.key])
- else:
- # send value as a lambda so that the value is
- # acquired after any autoflush occurs.
+ # use the "committed state" only if we're in a flush
+ # for this state.
+
+ sess = sessionlib._state_session(state)
+ if sess is not None and sess._flushing:
+ def visit_bindparam(bindparam):
+ if bindparam.key in bind_to_col:
bindparam.value = \
lambda: mapper._get_committed_state_attr_by_column(
state, dict_, bind_to_col[bindparam.key])
-
-
+ else:
+ def visit_bindparam(bindparam):
+ if bindparam.key in bind_to_col:
+ bindparam.value = lambda: mapper._get_state_attr_by_column(
+ state, dict_, bind_to_col[bindparam.key])
+
+
if self.parent_property.secondary is not None and alias_secondary:
criterion = sql_util.ClauseAdapter(
self.parent_property.secondary.alias()).\
criterion = visitors.cloned_traverse(
criterion, {}, {'bindparam':visit_bindparam})
+
if adapt_source:
criterion = adapt_source(criterion)
return criterion
return criterion
def _class_level_loader(self, state):
- if not state.has_identity:
+ if not state.has_identity and \
+ (not self.parent_property.load_on_pending or not state.session_id):
return None
return LoadLazyAttribute(state, self.key)
def __setstate__(self, state):
self.state, self.key = state
- def __call__(self, **kw):
+ def __call__(self, passive=False):
state = self.state
instance_mapper = mapper._state_mapper(state)
prop = instance_mapper.get_property(self.key)
strategy = prop._get_strategy(LazyLoader)
-
- if kw.get('passive') is attributes.PASSIVE_NO_FETCH and \
- not strategy.use_get:
+ pending = not state.key
+
+ if (
+ passive is attributes.PASSIVE_NO_FETCH and
+ not strategy.use_get
+ ) or (
+ passive is attributes.PASSIVE_ONLY_PERSISTENT and
+ pending
+ ):
return attributes.PASSIVE_NO_RESULT
-
+
if strategy._should_log_debug():
strategy.logger.debug("loading %s",
mapperutil.state_attribute_str(
q = session.query(prop.mapper)._adapt_all_clauses()
+ # don't autoflush on pending
+ # this would be something that's prominent in the
+ # docs and such
+ if pending:
+ q = q.autoflush(False)
+
if state.load_path:
q = q._with_current_path(state.load_path + (self.key,))
-
+
# if we have a simple primary key load, use mapper.get()
# to possibly save a DB round trip
if strategy.use_get:
ident = []
allnulls = True
+ if session._flushing:
+ get_attr = instance_mapper._get_committed_state_attr_by_column
+ else:
+ get_attr = instance_mapper._get_state_attr_by_column
+
+ # The many-to-one get is intended to be very fast. Note
+ # that we don't want to autoflush() if the get() doesn't
+ # actually have to hit the DB. It is now not necessary
+ # now that we use the pending attribute state.
for primary_key in prop.mapper.primary_key:
- val = instance_mapper.\
- _get_committed_state_attr_by_column(
+ val = get_attr(
state,
state.dict,
strategy._equated_columns[primary_key],
- **kw)
+ passive=passive)
if val is attributes.PASSIVE_NO_RESULT:
return val
allnulls = allnulls and val is None
q = q._conditional_options(*state.load_options)
key = prop.mapper.identity_key_from_primary_key(ident)
- return q._get(key, ident, **kw)
+ return q._get(key, ident, passive=passive)
if prop.order_by:
if state.load_options:
q = q._conditional_options(*state.load_options)
+
+ lazy_clause = strategy.lazy_clause(state)
+
+ if pending:
+ bind_values = sql_util.bind_values(lazy_clause)
+ if None in bind_values:
+ return None
- q = q.filter(strategy.lazy_clause(state))
+ q = q.filter(lazy_clause)
result = q.all()
if strategy.uselist:
return prop.compare(operators.eq,
instance,
- value_is_parent=True,
- detect_transient_pending=True)
+ value_is_parent=True)
def _entity_info(entity, compile=True):
visitors.traverse(clause, {}, {'column':cols.add})
return cols
+def bind_values(clause):
+ """Return an ordered list of "bound" values in the given clause.
+
+ E.g.::
+
+ >>> expr = and_(
+ ... table.c.foo==5, table.c.foo==7
+ ... )
+ >>> bind_values(expr)
+ [5, 7]
+ """
+
+ v = []
+ def visit_bindparam(bind):
+ value = bind.value
+
+ # evaluate callables
+ if callable(value):
+ value = value()
+
+ v.append(value)
+
+ visitors.traverse(clause, {}, {'bindparam':visit_bindparam})
+ return v
+
def _quote_ddl_expr(element):
if isinstance(element, basestring):
element = element.replace("'", "''")
--- /dev/null
+from sqlalchemy import *
+from sqlalchemy.orm import *
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.test.testing import TestBase, eq_, AssertsExecutionResults, assert_raises
+from sqlalchemy.test import testing
+from sqlalchemy.orm.attributes import instance_state
+from sqlalchemy.orm.exc import FlushError
+
+engine = testing.db
+
+
+class FlushOnPendingTest(AssertsExecutionResults, TestBase):
+ def setUp(self):
+ global Parent, Child, Base
+ Base= declarative_base()
+
+ class Parent(Base):
+ __tablename__ = 'parent'
+
+ id= Column(Integer, primary_key=True)
+ name = Column(String(50), nullable=False)
+ children = relationship("Child", load_on_pending=True)
+
+ class Child(Base):
+ __tablename__ = 'child'
+ id= Column(Integer, primary_key=True)
+ parent_id = Column(Integer, ForeignKey('parent.id'))
+
+ Base.metadata.create_all(engine)
+
+ def tearDown(self):
+ Base.metadata.drop_all(engine)
+
+ def test_annoying_autoflush_one(self):
+ sess = Session(engine)
+
+ p1 = Parent()
+ sess.add(p1)
+ p1.children = []
+
+ def test_annoying_autoflush_two(self):
+ sess = Session(engine)
+
+ p1 = Parent()
+ sess.add(p1)
+ assert p1.children == []
+
+ def test_dont_load_if_no_keys(self):
+ sess = Session(engine)
+
+ p1 = Parent()
+ sess.add(p1)
+
+ def go():
+ assert p1.children == []
+ self.assert_sql_count(testing.db, go, 0)
+
+class LoadOnFKsTest(AssertsExecutionResults, TestBase):
+
+ def setUp(self):
+ global Parent, Child, Base
+ Base= declarative_base()
+
+ class Parent(Base):
+ __tablename__ = 'parent'
+
+ id= Column(Integer, primary_key=True)
+
+ class Child(Base):
+ __tablename__ = 'child'
+ id= Column(Integer, primary_key=True)
+ parent_id = Column(Integer, ForeignKey('parent.id'))
+
+ parent = relationship(Parent, backref=backref("children"))
+
+ Base.metadata.create_all(engine)
+
+ global sess, p1, p2, c1, c2
+ sess = Session(bind=engine)
+
+ p1 = Parent()
+ p2 = Parent()
+ c1, c2 = Child(), Child()
+ c1.parent = p1
+ sess.add_all([p1, p2])
+ assert c1 in sess
+
+ sess.commit()
+
+ def tearDown(self):
+ Base.metadata.drop_all(engine)
+
+ def test_load_on_pending_disallows_backref_event(self):
+ Child.parent.property.load_on_pending = True
+ sess.autoflush = False
+ c3 = Child()
+ sess.add(c3)
+ c3.parent_id = p1.id
+ c3.parent = p1
+
+ # a side effect of load-on-pending with no autoflush.
+ # a change to the backref event handler to check
+ # collection membership before assuming "old == new so return"
+ # would fix this - but this is wasteful and autoflush
+ # should be turned on.
+ assert c3 not in p1.children
+
+ def test_load_on_persistent_allows_backref_event(self):
+ Child.parent.property.load_on_pending = True
+ c3 = Child()
+ sess.add(c3)
+ c3.parent_id = p1.id
+ c3.parent = p1
+
+ assert c3 in p1.children
+
+ def test_no_load_on_pending_allows_backref_event(self):
+ # users who stick with the program and don't use
+ # 'load_on_pending' get expected behavior
+
+ sess.autoflush = False
+ c3 = Child()
+ sess.add(c3)
+ c3.parent_id = p1.id
+
+ c3.parent = p1
+
+ assert c3 in p1.children
+
+ def test_load_on_pending_with_set(self):
+ Child.parent.property.load_on_pending = True
+
+ p1.children
+
+ c3 = Child()
+ sess.add(c3)
+
+ c3.parent_id = p1.id
+
+ def go():
+ c3.parent = p1
+ self.assert_sql_count(testing.db, go, 0)
+
+ def test_backref_doesnt_double(self):
+ Child.parent.property.load_on_pending = True
+ sess.autoflush = False
+ p1.children
+ c3 = Child()
+ sess.add(c3)
+ c3.parent = p1
+ c3.parent = p1
+ c3.parent = p1
+ c3.parent = p1
+ assert len(p1.children)== 2
+
+ def test_m2o_lazy_loader_on_persistent(self):
+ """Compare the behaviors from the lazyloader using
+ the "committed" state in all cases, vs. the lazyloader
+ using the "current" state in all cases except during flush.
+
+ """
+ for loadfk in (True, False):
+ for loadrel in (True, False):
+ for autoflush in (True, False):
+ for manualflush in (True, False):
+ for fake_autoexpire in (True, False):
+ sess.autoflush = autoflush
+
+ if loadfk:
+ c1.parent_id
+ if loadrel:
+ c1.parent
+
+ c1.parent_id = p2.id
+
+ if manualflush:
+ sess.flush()
+
+ # fake_autoexpire refers to the eventual
+ # auto-expire of 'parent' when c1.parent_id
+ # is altered.
+ if fake_autoexpire:
+ sess.expire(c1, ['parent'])
+
+ # old 0.6 behavior
+ #if manualflush and (not loadrel or fake_autoexpire):
+ # # a flush occurs, we get p2
+ # assert c1.parent is p2
+ #elif not loadrel and not loadfk:
+ # # problematically - we get None since committed state
+ # # is empty when c1.parent_id was mutated, since we want
+ # # to save on selects. this is
+ # # why the patch goes in in 0.6 - this is mostly a bug.
+ # assert c1.parent is None
+ #else:
+ # # if things were loaded, autoflush doesn't even
+ # # happen.
+ # assert c1.parent is p1
+
+ # new behavior
+ if loadrel and not fake_autoexpire:
+ assert c1.parent is p1
+ else:
+ assert c1.parent is p2
+
+ sess.rollback()
+
+ def test_m2o_lazy_loader_on_pending(self):
+ for loadonpending in (False, True):
+ for autoflush in (False, True):
+ for manualflush in (False, True):
+ Child.parent.property.load_on_pending = loadonpending
+ sess.autoflush = autoflush
+ c2 = Child()
+ sess.add(c2)
+ c2.parent_id = p2.id
+
+ if manualflush:
+ sess.flush()
+
+ if loadonpending or manualflush:
+ assert c2.parent is p2
+ else:
+ assert c2.parent is None
+
+ sess.rollback()
+
+ def test_m2o_lazy_loader_on_transient(self):
+ for loadonpending in (False, True):
+ for attach in (False, True):
+ for autoflush in (False, True):
+ for manualflush in (False, True):
+ Child.parent.property.load_on_pending = loadonpending
+ sess.autoflush = autoflush
+ c2 = Child()
+
+ if attach:
+ sess._attach(instance_state(c2))
+
+ c2.parent_id = p2.id
+
+ if manualflush:
+ sess.flush()
+
+ if loadonpending and attach:
+ assert c2.parent is p2
+ else:
+ assert c2.parent is None
+
+ sess.rollback()
clear_mappers()
super(SavePostTest, cls).teardown_class()
- def testattach(self):
- """test that a transient/pending instance has proper bi-directional behavior.
+ def test_attach_noautoflush(self):
+ s = create_session(bind=testing.db, autoflush=False)
- this requires that lazy loaders do not fire off for a transient/pending instance."""
- s = create_session(bind=testing.db)
+ s.begin()
+ try:
+ blog = s.query(Blog).get(blog_id)
+ user = s.query(User).get(user_id)
+ post = Post(headline="asdf asdf", summary="asdfasfd", user=user)
+ s.add(post)
+ post.blog_id=blog_id
+ post.blog = blog
+ assert post in blog.posts
+ finally:
+ s.rollback()
+
+ def test_attach_autoflush(self):
+ s = create_session(bind=testing.db, autoflush=True)
s.begin()
try:
blog = s.query(Blog).get(blog_id)
- post = Post(headline="asdf asdf", summary="asdfasfd")
+ user = s.query(User).get(user_id)
+ post = Post(headline="asdf asdf", summary="asdfasfd", user=user)
s.add(post)
post.blog_id=blog_id
post.blog = blog
projects / thirdparty / sqlalchemy / sqlalchemy.git / commitdiff
