From: Mike Bayer Date: Sun, 12 Dec 2010 18:01:34 +0000 (-0500) Subject: - inlinings and callcount reductions X-Git-Tag: rel_0_7b1~172 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0d71ea8126137d2b3d4141aa0fb30c2e64376d44;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - inlinings and callcount reductions - add test coverage for the rare case of noload->lazyload + pickle --- diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index 0e9e6739c0..63331e0813 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -1018,7 +1018,7 @@ def clear_mappers(): def joinedload(*keys, **kw): """Return a ``MapperOption`` that will convert the property of the given - name into an joined eager load. + name or series of mapped attributes into an joined eager load. .. note:: This function is known as :func:`eagerload` in all versions of SQLAlchemy prior to version 0.6beta3, including the 0.5 and 0.4 @@ -1065,7 +1065,8 @@ def joinedload(*keys, **kw): def joinedload_all(*keys, **kw): """Return a ``MapperOption`` that will convert all properties along the - given dot-separated path into an joined eager load. + given dot-separated path or series of mapped attributes + into an joined eager load. .. note:: This function is known as :func:`eagerload_all` in all versions of SQLAlchemy prior to version 0.6beta3, including the 0.5 and 0.4 @@ -1111,7 +1112,8 @@ def eagerload_all(*args, **kwargs): def subqueryload(*keys): """Return a ``MapperOption`` that will convert the property - of the given name into an subquery eager load. + of the given name or series of mapped attributes + into an subquery eager load. Used with :meth:`~sqlalchemy.orm.query.Query.options`. @@ -1135,7 +1137,8 @@ def subqueryload(*keys): def subqueryload_all(*keys): """Return a ``MapperOption`` that will convert all properties along the - given dot-separated path into a subquery eager load. + given dot-separated path or series of mapped attributes + into a subquery eager load. Used with :meth:`~sqlalchemy.orm.query.Query.options`. @@ -1158,7 +1161,7 @@ def subqueryload_all(*keys): def lazyload(*keys): """Return a ``MapperOption`` that will convert the property of the given - name into a lazy load. + name or series of mapped attributes into a lazy load. Used with :meth:`~sqlalchemy.orm.query.Query.options`. @@ -1167,9 +1170,21 @@ def lazyload(*keys): """ return strategies.EagerLazyOption(keys, lazy=True) +def lazyload_all(*keys): + """Return a ``MapperOption`` that will convert all the properties + along the given dot-separated path or series of mapped attributes + into a lazy load. + + Used with :meth:`~sqlalchemy.orm.query.Query.options`. + + See also: :func:`eagerload`, :func:`subqueryload`, :func:`immediateload` + + """ + return strategies.EagerLazyOption(keys, lazy=True, chained=True) + def noload(*keys): """Return a ``MapperOption`` that will convert the property of the - given name into a non-load. + given name or series of mapped attributes into a non-load. Used with :meth:`~sqlalchemy.orm.query.Query.options`. @@ -1180,7 +1195,7 @@ def noload(*keys): def immediateload(*keys): """Return a ``MapperOption`` that will convert the property of the given - name into an immediate load. + name or series of mapped attributes into an immediate load. Used with :meth:`~sqlalchemy.orm.query.Query.options`. diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py index 4d03795036..002215268a 100644 --- a/lib/sqlalchemy/orm/attributes.py +++ b/lib/sqlalchemy/orm/attributes.py @@ -109,7 +109,7 @@ class QueryableAttribute(interfaces.PropComparator): def __str__(self): return repr(self.parententity) + "." + self.property.key - @property + @util.memoized_property def property(self): return self.comparator.property @@ -310,14 +310,6 @@ class AttributeImpl(object): def get_all_pending(self, state, dict_): raise NotImplementedError() - def _get_callable(self, state): - if self.key in state.callables: - return state.callables[self.key] - elif self.callable_ is not None: - return self.callable_(state) - else: - return None - def initialize(self, state, dict_): """Initialize the given state's attribute with an empty value.""" @@ -340,7 +332,13 @@ class AttributeImpl(object): if passive is PASSIVE_NO_INITIALIZE: return PASSIVE_NO_RESULT - callable_ = self._get_callable(state) + if self.key in state.callables: + callable_ = state.callables[self.key] + elif self.callable_ is not None: + callable_ = self.callable_(state) + else: + callable_ = None + if callable_ is not None: #if passive is not PASSIVE_OFF: # return PASSIVE_NO_RESULT @@ -370,21 +368,19 @@ class AttributeImpl(object): """return the unchanged value of this attribute""" if self.key in state.committed_state: - if state.committed_state[self.key] is NO_VALUE: + value = state.committed_state[self.key] + if value is NO_VALUE: return None else: - return state.committed_state.get(self.key) + return value else: return self.get(state, dict_, passive=passive) def set_committed_value(self, state, dict_, value): """set an attribute value on the given instance and 'commit' it.""" + dict_[self.key] = value state.commit(dict_, [self.key]) - - state.callables.pop(self.key, None) - state.dict[self.key] = value - return value class ScalarAttributeImpl(AttributeImpl): diff --git a/lib/sqlalchemy/orm/dependency.py b/lib/sqlalchemy/orm/dependency.py index b9958f7758..19c78c5c84 100644 --- a/lib/sqlalchemy/orm/dependency.py +++ b/lib/sqlalchemy/orm/dependency.py @@ -844,7 +844,7 @@ class DetectKeySwitch(DependencyProcessor): uowcommit, self.passive_updates) def _pks_changed(self, uowcommit, state): - return state.has_identity and sync.source_modified(uowcommit, + return bool(state.key) and sync.source_modified(uowcommit, state, self.mapper, self.prop.synchronize_pairs) diff --git a/lib/sqlalchemy/orm/dynamic.py b/lib/sqlalchemy/orm/dynamic.py index 710b710a91..4637bad7e5 100644 --- a/lib/sqlalchemy/orm/dynamic.py +++ b/lib/sqlalchemy/orm/dynamic.py @@ -199,7 +199,7 @@ class AppenderMixin(object): self.attr = attr mapper = object_mapper(instance) - prop = mapper.get_property(self.attr.key) + prop = mapper._props[self.attr.key] self._criterion = prop.compare( operators.eq, instance, diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index cd9f01f38e..dc0799d147 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -872,7 +872,7 @@ class Mapper(object): for mapper in self.iterate_to_root(): for (key, cls) in mapper.delete_orphans: if attributes.manager_of_class(cls).has_parent( - state, key, optimistic=state.has_identity): + state, key, optimistic=bool(state.key)): return False o = o or bool(mapper.delete_orphans) return o @@ -1582,7 +1582,7 @@ class Mapper(object): else: conn = connection - has_identity = state.has_identity + has_identity = bool(state.key) mapper = _state_mapper(state) instance_key = state.key or mapper._identity_key_from_state(state) @@ -1998,7 +1998,7 @@ class Mapper(object): tups.append((state, state.dict, _state_mapper(state), - state.has_identity, + bool(state.key), conn)) table_to_mapper = self._sorted_tables @@ -2503,7 +2503,7 @@ def _load_scalar_attributes(state, attribute_names): "attribute refresh operation cannot proceed" % (state_str(state))) - has_key = state.has_identity + has_key = bool(state.key) result = False diff --git a/lib/sqlalchemy/orm/state.py b/lib/sqlalchemy/orm/state.py index 278f86749b..48861085d3 100644 --- a/lib/sqlalchemy/orm/state.py +++ b/lib/sqlalchemy/orm/state.py @@ -375,10 +375,14 @@ class InstanceState(object): """ class_manager = self.manager - for key in keys: - if key in dict_ and key in class_manager.mutable_attributes: - self.committed_state[key] = self.manager[key].impl.copy(dict_[key]) - else: + if class_manager.mutable_attributes: + for key in keys: + if key in dict_ and key in class_manager.mutable_attributes: + self.committed_state[key] = self.manager[key].impl.copy(dict_[key]) + else: + self.committed_state.pop(key, None) + else: + for key in keys: self.committed_state.pop(key, None) self.expired = False diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index a6711ae261..d6fb0c005f 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -245,7 +245,7 @@ class DeferredColumnLoader(LoaderStrategy): path, adapter, **kwargs) def _class_level_loader(self, state): - if not state.has_identity: + if not state.key: return None return LoadDeferredColumns(state, self.key) @@ -255,19 +255,28 @@ log.class_logger(DeferredColumnLoader) class LoadDeferredColumns(object): """serializable loader object used by DeferredColumnLoader""" + + __slots__ = 'state', 'key' def __init__(self, state, key): - self.state, self.key = state, key - + self.state = state + self.key = key + + def __getstate__(self): + return self.state, self.key + + def __setstate__(self, state): + self.state, self.key = state + def __call__(self, passive=False): + state, key = self.state, self.key + if passive is attributes.PASSIVE_NO_FETCH: return attributes.PASSIVE_NO_RESULT - state = self.state - localparent = mapper._state_mapper(state) - prop = localparent.get_property(self.key) + prop = localparent._props[key] strategy = prop._get_strategy(DeferredColumnLoader) if strategy.group: @@ -279,7 +288,7 @@ class LoadDeferredColumns(object): p.group==strategy.group ] else: - toload = [self.key] + toload = [key] # narrow the keys down to just those which have no history group = [k for k in toload if k in state.unmodified] @@ -289,7 +298,7 @@ class LoadDeferredColumns(object): raise orm_exc.DetachedInstanceError( "Parent instance %s is not bound to a Session; " "deferred load operation of attribute '%s' cannot proceed" % - (mapperutil.state_str(state), self.key) + (mapperutil.state_str(state), key) ) query = session.query(localparent) @@ -475,7 +484,7 @@ class LazyLoader(AbstractRelationshipLoader): return criterion def _class_level_loader(self, state): - if not state.has_identity and \ + if not state.key and \ (not self.parent_property.load_on_pending or not state.session_id): return None @@ -556,20 +565,23 @@ log.class_logger(LazyLoader) class LoadLazyAttribute(object): """serializable loader object used by LazyLoader""" - + + __slots__ = 'state', 'key' + def __init__(self, state, key): - self.state, self.key = state, key - + self.state = state + self.key = key + def __getstate__(self): - return (self.state, self.key) - + return self.state, self.key + def __setstate__(self, state): self.state, self.key = state - + def __call__(self, passive=False): - state = self.state + state, key = self.state, self.key instance_mapper = mapper._state_mapper(state) - prop = instance_mapper.get_property(self.key) + prop = instance_mapper._props[key] strategy = prop._get_strategy(LazyLoader) pending = not state.key @@ -587,7 +599,7 @@ class LoadLazyAttribute(object): raise orm_exc.DetachedInstanceError( "Parent instance %s is not bound to a Session; " "lazy load operation of attribute '%s' cannot proceed" % - (mapperutil.state_str(state), self.key) + (mapperutil.state_str(state), key) ) # if we have a simple primary key load, check the @@ -612,8 +624,8 @@ class LoadLazyAttribute(object): if _none_set.issuperset(ident): return None - key = prop.mapper.identity_key_from_primary_key(ident) - instance = Query._get_from_identity(session, key, passive) + ident_key = prop.mapper.identity_key_from_primary_key(ident) + instance = Query._get_from_identity(session, ident_key, passive) if instance is not None: return instance elif passive is attributes.PASSIVE_NO_FETCH: @@ -626,13 +638,13 @@ class LoadLazyAttribute(object): q = q.autoflush(False) if state.load_path: - q = q._with_current_path(state.load_path + (self.key,)) + q = q._with_current_path(state.load_path + (key,)) if state.load_options: q = q._conditional_options(*state.load_options) if strategy.use_get: - return q._load_on_ident(key) + return q._load_on_ident(ident_key) if prop.order_by: q = q.order_by(*util.to_list(prop.order_by)) @@ -736,7 +748,7 @@ class SubqueryLoader(AbstractRelationshipLoader): else: leftmost_mapper, leftmost_prop = \ subq_mapper, \ - subq_mapper.get_property(subq_path[1]) + subq_mapper._props[subq_path[1]] leftmost_cols, remote_cols = self._local_remote_columns(leftmost_prop) leftmost_attr = [ @@ -1272,7 +1284,7 @@ class LoadEagerFromAliasOption(PropertyOption): if isinstance(self.alias, basestring): mapper = mappers[-1] (root_mapper, propname) = paths[-1][-2:] - prop = mapper.get_property(propname) + prop = mapper._props[propname] self.alias = prop.target.alias(self.alias) query._attributes[ ("user_defined_eager_row_processor", @@ -1281,7 +1293,7 @@ class LoadEagerFromAliasOption(PropertyOption): else: (root_mapper, propname) = paths[-1][-2:] mapper = mappers[-1] - prop = mapper.get_property(propname) + prop = mapper._props[propname] adapter = query._polymorphic_adapters.get(prop.mapper, None) query._attributes[ ("user_defined_eager_row_processor", diff --git a/lib/sqlalchemy/orm/unitofwork.py b/lib/sqlalchemy/orm/unitofwork.py index d9d64fe391..ab62e5324c 100644 --- a/lib/sqlalchemy/orm/unitofwork.py +++ b/lib/sqlalchemy/orm/unitofwork.py @@ -35,7 +35,7 @@ class UOWEventHandler(interfaces.AttributeExtension): sess = session._state_session(state) if sess: - prop = _state_mapper(state).get_property(self.key) + prop = _state_mapper(state)._props[self.key] if prop.cascade.save_update and \ (prop.cascade_backrefs or self.key == initiator.key) and \ item not in sess: @@ -45,7 +45,7 @@ class UOWEventHandler(interfaces.AttributeExtension): def remove(self, state, item, initiator): sess = session._state_session(state) if sess: - prop = _state_mapper(state).get_property(self.key) + prop = _state_mapper(state)._props[self.key] # expunge pending orphans if prop.cascade.delete_orphan and \ item in sess.new and \ @@ -60,7 +60,7 @@ class UOWEventHandler(interfaces.AttributeExtension): sess = session._state_session(state) if sess: - prop = _state_mapper(state).get_property(self.key) + prop = _state_mapper(state)._props[self.key] if newvalue is not None and \ prop.cascade.save_update and \ (prop.cascade_backrefs or self.key == initiator.key) and \ diff --git a/lib/sqlalchemy/orm/util.py b/lib/sqlalchemy/orm/util.py index c59dbed692..b5fa0c0cff 100644 --- a/lib/sqlalchemy/orm/util.py +++ b/lib/sqlalchemy/orm/util.py @@ -12,6 +12,7 @@ from sqlalchemy.orm.interfaces import MapperExtension, EXT_CONTINUE,\ PropComparator, MapperProperty,\ AttributeExtension from sqlalchemy.orm import attributes, exc +import operator mapperlib = util.importlater("sqlalchemy.orm", "mapperlib") @@ -514,8 +515,7 @@ def _attr_as_key(attr): def _is_aliased_class(entity): return isinstance(entity, AliasedClass) -def _state_mapper(state): - return state.manager.mapper +_state_mapper = util.dottedgetter('manager.mapper') def object_mapper(instance): """Given an object, return the primary Mapper associated with the object diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index aa150874fc..9119e35b78 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -6,7 +6,7 @@ from compat import callable, cmp, reduce, defaultdict, py25_dict, \ threading, py3k, jython, win32, set_types, buffer, pickle, \ - update_wrapper, partial, md5_hex, decode_slice + update_wrapper, partial, md5_hex, decode_slice, dottedgetter from _collections import NamedTuple, ImmutableContainer, frozendict, \ Properties, OrderedProperties, ImmutableProperties, OrderedDict, \ diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index 59dd9eaf08..79dd6228f8 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -188,6 +188,16 @@ else: def decode_slice(slc): return (slc.start, slc.stop, slc.step) +if sys.version_info >= (2, 6): + from operator import attrgetter as dottedgetter +else: + def dottedgetter(attr): + def g(obj): + for name in attr.split("."): + obj = getattr(obj, name) + return obj + return g + import decimal diff --git a/test/aaa_profiling/test_orm.py b/test/aaa_profiling/test_orm.py index 41d15d5cea..a5bdc6ad60 100644 --- a/test/aaa_profiling/test_orm.py +++ b/test/aaa_profiling/test_orm.py @@ -53,16 +53,16 @@ class MergeTest(_base.MappedTest): # down from 185 on this this is a small slice of a usually # bigger operation so using a small variance - @profiling.function_call_count(91, variance=0.05, - versions={'2.4': 68, '3': 89}) + @profiling.function_call_count(86, variance=0.05, + versions={'2.4': 68, '2.5':94, '3': 89}) def go(): return sess2.merge(p1, load=False) p2 = go() # third call, merge object already present. almost no calls. - @profiling.function_call_count(12, variance=0.05, - versions={'2.4': 8, '3': 13}) + @profiling.function_call_count(11, variance=0.05, + versions={'2.4': 8, '2.5':15, '3': 13}) def go(): return sess2.merge(p2, load=False) p3 = go() @@ -79,7 +79,7 @@ class MergeTest(_base.MappedTest): # using sqlite3 the C extension took it back up to approx. 1257 # (py2.6) - @profiling.function_call_count(1257, + @profiling.function_call_count(1194, versions={'2.5':1191, '2.6':1191, '2.6+cextension':1194, '2.4': 807} @@ -103,8 +103,12 @@ class LoadManyToOneFromIdentityTest(_base.MappedTest): """ - # 2.4's profiler has different callcounts - __skip_if__ = lambda : sys.version_info < (2, 5), + # only need to test for unexpected variance in a large call + # count here, + # so remove some platforms that have wildly divergent + # callcounts. + __requires__ = 'python25', + __unsupported_on__ = 'postgresql+pg8000', @classmethod def define_tables(cls, metadata): @@ -168,7 +172,7 @@ class LoadManyToOneFromIdentityTest(_base.MappedTest): parents = sess.query(Parent).all() children = sess.query(Child).all() - @profiling.function_call_count(33977) + @profiling.function_call_count(23979, {'2.5':28974}) def go(): for p in parents: p.child diff --git a/test/aaa_profiling/test_zoomark.py b/test/aaa_profiling/test_zoomark.py index 7108f4c94b..85e4161840 100644 --- a/test/aaa_profiling/test_zoomark.py +++ b/test/aaa_profiling/test_zoomark.py @@ -27,7 +27,6 @@ class ZooMarkTest(TestBase): """ __only_on__ = 'postgresql+psycopg2' - __skip_if__ = lambda : sys.version_info < (2, 4), def test_baseline_0_setup(self): global metadata diff --git a/test/aaa_profiling/test_zoomark_orm.py b/test/aaa_profiling/test_zoomark_orm.py index 66975ffde9..ba37eaef94 100644 --- a/test/aaa_profiling/test_zoomark_orm.py +++ b/test/aaa_profiling/test_zoomark_orm.py @@ -335,11 +335,11 @@ class ZooMarkTest(TestBase): def test_profile_1_create_tables(self): self.test_baseline_1_create_tables() - @profiling.function_call_count(7321) + @profiling.function_call_count(6891) def test_profile_1a_populate(self): self.test_baseline_1a_populate() - @profiling.function_call_count(507) + @profiling.function_call_count(481) def test_profile_2_insert(self): self.test_baseline_2_insert() diff --git a/test/lib/profiling.py b/test/lib/profiling.py index eeb3901cb3..6216e75b17 100644 --- a/test/lib/profiling.py +++ b/test/lib/profiling.py @@ -14,6 +14,9 @@ __all__ = 'profiled', 'function_call_count', 'conditional_call_count' all_targets = set() profile_config = { 'targets': set(), 'report': True, + 'print_callers':False, + 'print_callees':False, + 'graphic':False, 'sort': ('time', 'calls'), 'limit': None } profiler = None @@ -47,21 +50,35 @@ def profiled(target=None, **target_opts): elapsed, load_stats, result = _profile( filename, fn, *args, **kw) - - report = target_opts.get('report', profile_config['report']) - if report: - sort_ = target_opts.get('sort', profile_config['sort']) - limit = target_opts.get('limit', profile_config['limit']) - print "Profile report for target '%s' (%s)" % ( - target, filename) - - stats = load_stats() - stats.sort_stats(*sort_) - if limit: - stats.print_stats(limit) - else: - stats.print_stats() - #stats.print_callers() + + graphic = target_opts.get('graphic', profile_config['graphic']) + if graphic: + os.system("runsnake %s" % filename) + else: + report = target_opts.get('report', profile_config['report']) + if report: + sort_ = target_opts.get('sort', profile_config['sort']) + limit = target_opts.get('limit', profile_config['limit']) + print "Profile report for target '%s' (%s)" % ( + target, filename) + + stats = load_stats() + stats.sort_stats(*sort_) + if limit: + stats.print_stats(limit) + else: + stats.print_stats() + + print_callers = target_opts.get('print_callers', + profile_config['print_callers']) + if print_callers: + stats.print_callers() + + print_callees = target_opts.get('print_callees', + profile_config['print_callees']) + if print_callees: + stats.print_callees() + os.unlink(filename) return result return decorate diff --git a/test/orm/test_pickled.py b/test/orm/test_pickled.py index a246ddbdcd..972b298af2 100644 --- a/test/orm/test_pickled.py +++ b/test/orm/test_pickled.py @@ -8,7 +8,8 @@ from test.lib.schema import Table, Column from sqlalchemy.orm import mapper, relationship, create_session, \ sessionmaker, attributes, interfaces,\ clear_mappers, exc as orm_exc,\ - configure_mappers + configure_mappers, Session, lazyload_all,\ + lazyload from test.orm import _base, _fixtures @@ -127,6 +128,33 @@ class PickleTest(_fixtures.FixtureTest): eq_(u2.name, 'ed') eq_(u2, User(name='ed', addresses=[Address(email_address='ed@bar.com')])) + @testing.resolve_artifact_names + def test_instance_lazy_relation_loaders(self): + mapper(User, users, properties={ + 'addresses':relationship(Address, lazy='noload') + }) + mapper(Address, addresses) + + sess = Session() + u1 = User(name='ed', addresses=[ + Address( + email_address='ed@bar.com', + ) + ]) + + sess.add(u1) + sess.commit() + sess.close() + + u1 = sess.query(User).options( + lazyload(User.addresses) + ).first() + u2 = pickle.loads(pickle.dumps(u1)) + + sess = Session() + sess.add(u2) + assert u2.addresses + @testing.resolve_artifact_names def test_instance_deferred_cols(self): mapper(User, users, properties={