From: Mike Bayer Date: Sat, 26 May 2007 16:21:39 +0000 (+0000) Subject: some change to populate_instance etc., allows poly secondary load to re-use popoulate... X-Git-Tag: rel_0_4_6~241 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=108f17c06b67211f9bbf2336bd3c6cc3b315685f;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git some change to populate_instance etc., allows poly secondary load to re-use popoulate_instance --- diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 35f67c5ec6..e642bd54bf 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -23,19 +23,25 @@ class MapperProperty(object): """return a tuple of a row processing and a row post-processing function. Input arguments are the query.SelectionContext and the *first* - row of a result set obtained within query.Query.instances(). + applicable row of a result set obtained within query.Query.instances(), called + the first time mapper.populate_instance() is invoked for a particular + result, and only once per result. + By looking at the columns present within the row, MapperProperty returns two callables which will be used to process the instance that results from the row. callables are of the following form: - def execute(instance, row, identitykey, isnew): - # process incoming instance, given row, identitykey, - # isnew flag indicating if this is the first row corresponding to this - # instance + def execute(instance, row, flags): + # process incoming instance and given row. + # flags is a dictionary containing at least the following attributes: + # isnew - indicates if the instance was newly created as a result of reading this row + # instancekey - identity key of the instance + # optional attribute: + # ispostselect - indicates if this row resulted from a 'post' select of additional tables/columns - def post_execute(instance): + def post_execute(instance, flags): # process instance after all result rows have been processed. this # function should be used to issue additional selections in order to # eagerly load additional properties. @@ -47,7 +53,6 @@ class MapperProperty(object): """ raise NotImplementedError() - def cascade_iterator(self, type, object, recursive=None, halt_on=None): return [] diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index d0de95dda5..24fdb8e6cc 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1462,9 +1462,9 @@ class Mapper(object): if not context.identity_map.has_key(identitykey): context.identity_map[identitykey] = instance isnew = True - if extension.populate_instance(self, context, row, instance, identitykey, isnew) is EXT_PASS: - self.populate_instance(context, instance, row, identitykey, isnew) - if extension.append_result(self, context, row, instance, identitykey, result, isnew) is EXT_PASS: + if extension.populate_instance(self, context, row, instance, {'instancekey':identitykey, 'isnew':isnew}) is EXT_PASS: + self.populate_instance(context, instance, row, {'instancekey':identitykey, 'isnew':isnew}) + if extension.append_result(self, context, row, instance, result, {'instancekey':identitykey, 'isnew':isnew}) is EXT_PASS: if result is not None: result.append(instance) return instance @@ -1505,9 +1505,9 @@ class Mapper(object): # call further mapper properties on the row, to pull further # instances from the row and possibly populate this item. - if extension.populate_instance(self, context, row, instance, identitykey, isnew) is EXT_PASS: - self.populate_instance(context, instance, row, identitykey, isnew) - if extension.append_result(self, context, row, instance, identitykey, result, isnew) is EXT_PASS: + if extension.populate_instance(self, context, row, instance, {'instancekey':identitykey, 'isnew':isnew}) is EXT_PASS: + self.populate_instance(context, instance, row, {'instancekey':identitykey, 'isnew':isnew}) + if extension.append_result(self, context, row, instance, result, {'instancekey':identitykey, 'isnew':isnew}) is EXT_PASS: if result is not None: result.append(instance) return instance @@ -1555,10 +1555,10 @@ class Mapper(object): newrow[c] = row[c2] return newrow - def populate_instance(self, selectcontext, instance, row, identitykey, isnew): + def populate_instance(self, selectcontext, instance, row, flags): """populate an instance from a result row.""" - populators = selectcontext.attributes.get(('instance_populators', self), None) + populators = selectcontext.attributes.get(('instance_populators', self, flags.get('ispostselect')), None) if populators is None: populators = [] post_processors = [] @@ -1573,19 +1573,19 @@ class Mapper(object): if poly_select_loader is not None: post_processors.append(poly_select_loader) - selectcontext.attributes[('instance_populators', self)] = populators - selectcontext.attributes[('post_processors', self)] = post_processors + selectcontext.attributes[('instance_populators', self, flags.get('ispostselect'))] = populators + selectcontext.attributes[('post_processors', self, flags.get('ispostselect'))] = post_processors for p in populators: - p(instance, row, identitykey, isnew) + p(instance, row, flags) if self.non_primary: selectcontext.attributes[('populating_mapper', instance)] = self def _post_instance(self, selectcontext, instance): - post_processors = selectcontext.attributes[('post_processors', self)] + post_processors = selectcontext.attributes[('post_processors', self, None)] for p in post_processors: - p(instance) + p(instance, {}) def _get_poly_select_loader(self, selectcontext, row): # 'select' or 'union'+col not present @@ -1593,14 +1593,9 @@ class Mapper(object): if hosted_mapper is None or len(needs_tables)==0 or hosted_mapper.polymorphic_fetch == 'deferred': return - from strategies import ColumnLoader - from attributes import InstrumentedAttribute - cond, param_names = self._deferred_inheritance_condition(needs_tables) statement = sql.select(needs_tables, cond, use_labels=True) - group = [p for p in self.props.values() if isinstance(p.strategy, ColumnLoader) and p.columns[0].table in needs_tables] - - def post_execute(instance): + def post_execute(instance, flags): self.__log_debug("Post query loading instance " + mapperutil.instance_str(instance)) identitykey = self.instance_key(instance) @@ -1609,8 +1604,8 @@ class Mapper(object): for c in param_names: params[c.name] = self.get_attr_by_column(instance, c) row = selectcontext.session.connection(self).execute(statement, **params).fetchone() - for prop in group: - InstrumentedAttribute.get_instrument(instance, prop.key).set_committed_value(instance, row[prop.columns[0]]) + self.populate_instance(selectcontext, instance, row, {'isnew':False, 'instancekey':identitykey, 'ispostselect':True}) + return post_execute Mapper.logger = logging.class_logger(Mapper) @@ -1715,7 +1710,7 @@ class MapperExtension(object): return EXT_PASS - def append_result(self, mapper, selectcontext, row, instance, identitykey, result, isnew): + def append_result(self, mapper, selectcontext, row, instance, result, flags): """Receive an object instance before that instance is appended to a result list. @@ -1737,22 +1732,17 @@ class MapperExtension(object): instance The object instance to be appended to the result. - identitykey - The identity key of the instance. - result List to which results are being appended. - isnew - Indicates if this is the first time we have seen this object - instance in the current result set. if you are selecting - from a join, such as an eager load, you might see the same - object instance many times in the same result set. + flags + extra information about the row, same as criterion in + `create_row_processor()` method of [sqlalchemy.orm.interfaces#MapperProperty] """ return EXT_PASS - def populate_instance(self, mapper, selectcontext, row, instance, identitykey, isnew): + def populate_instance(self, mapper, selectcontext, row, instance, flags): """Receive a newly-created instance before that instance has its attributes populated. diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 3af1bb9e31..61a37d435f 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -1123,8 +1123,10 @@ class SelectionContext(OperationContext): yet been added as persistent to the Session. attributes - A dictionary to store arbitrary data; eager loaders use it to - store additional result lists. + A dictionary to store arbitrary data; mappers, strategies, and + options all store various state information here in order + to communicate with each other and to themselves. + populate_existing Indicates if its OK to overwrite the attributes of instances diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index e1190c5a36..2941815a1c 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -35,8 +35,8 @@ class ColumnLoader(LoaderStrategy): def create_row_processor(self, selectcontext, mapper, row): if self.columns[0] in row: - def execute(instance, row, identitykey, isnew): - if isnew: + def execute(instance, row, flags): + if flags['isnew'] or flags.get('ispostselect'): if self._should_log_debug: self.logger.debug("populating %s with %s/%s" % (mapperutil.attribute_str(instance, self.key), row.__class__.__name__, self.columns[0].key)) instance.__dict__[self.key] = row[self.columns[0]] @@ -48,14 +48,15 @@ class ColumnLoader(LoaderStrategy): return (None, None) if hosted_mapper.polymorphic_fetch == 'deferred': - def execute(instance, row, identitykey, isnew): - sessionlib.attribute_manager.init_instance_attribute(instance, self.key, False, callable_=self._get_deferred_loader(instance, mapper, needs_tables)) + def execute(instance, row, flags): + if flags['isnew']: + sessionlib.attribute_manager.init_instance_attribute(instance, self.key, False, callable_=self._get_deferred_loader(instance, mapper, needs_tables)) self.logger.debug("Returning deferred column fetcher for %s %s" % (mapper, self.key)) return (execute, None) else: self.logger.debug("Returning no column fetcher for %s %s" % (mapper, self.key)) return (None, None) - + def _get_deferred_loader(self, instance, mapper, needs_tables): def load(): group = [p for p in mapper.props.values() if isinstance(p.strategy, ColumnLoader) and p.columns[0].table in needs_tables] @@ -95,20 +96,18 @@ class DeferredColumnLoader(LoaderStrategy): def create_row_processor(self, selectcontext, mapper, row): if not self.is_default or len(selectcontext.options): - def execute(instance, row, identitykey, isnew): - if not isnew: - return - if self._should_log_debug: - self.logger.debug("set deferred callable on %s" % mapperutil.attribute_str(instance, self.key)) - sessionlib.attribute_manager.init_instance_attribute(instance, self.key, False, callable_=self.setup_loader(instance)) + def execute(instance, row, flags): + if flags['isnew']: + if self._should_log_debug: + self.logger.debug("set deferred callable on %s" % mapperutil.attribute_str(instance, self.key)) + sessionlib.attribute_manager.init_instance_attribute(instance, self.key, False, callable_=self.setup_loader(instance)) return (execute, None) else: - def execute(instance, row, identitykey, isnew): - if not isnew: - return - if self._should_log_debug: - self.logger.debug("set deferred callable on %s" % mapperutil.attribute_str(instance, self.key)) - sessionlib.attribute_manager.reset_instance_attribute(instance, self.key) + def execute(instance, row, flags): + if flags['isnew']: + if self._should_log_debug: + self.logger.debug("set deferred callable on %s" % mapperutil.attribute_str(instance, self.key)) + sessionlib.attribute_manager.reset_instance_attribute(instance, self.key) return (execute, None) def init(self): @@ -123,11 +122,6 @@ class DeferredColumnLoader(LoaderStrategy): def setup_query(self, context, **kwargs): pass - - - def _load_deferred_tables(self, context, instance, loadtype): - (hosted_mapper, needs_tables) = context.attributes[('polymorphic_fetch', localparent, loadtype)] - group = [p for p in localparent.props.values() if isinstance(p.strategy, ColumnLoader) and p.columns[0].table in needs_tables] def setup_loader(self, instance): localparent = mapper.object_mapper(instance, raiseerror=False) @@ -302,8 +296,8 @@ class LazyLoader(AbstractRelationLoader): def create_row_processor(self, selectcontext, mapper, row): if not self.is_default or len(selectcontext.options): - def execute(instance, row, identitykey, isnew): - if isnew: + def execute(instance, row, flags): + if flags['isnew']: if self._should_log_debug: self.logger.debug("set instance-level lazy loader on %s" % mapperutil.attribute_str(instance, self.key)) # we are not the primary manager for this attribute on this class - set up a per-instance lazyloader, @@ -311,8 +305,8 @@ class LazyLoader(AbstractRelationLoader): self._init_instance_attribute(instance, callable_=self.setup_loader(instance, selectcontext.options)) return (execute, None) else: - def execute(instance, row, identitykey, isnew): - if isnew: + def execute(instance, row, flags): + if flags['isnew']: if self._should_log_debug: self.logger.debug("set class-level lazy loader on %s" % mapperutil.attribute_str(instance, self.key)) # we are the primary manager for this attribute on this class - reset its per-instance attribute state, @@ -627,7 +621,7 @@ class EagerLoader(AbstractRelationLoader): def create_row_processor(self, selectcontext, mapper, row): row_decorator = self._create_row_decorator(selectcontext, row) if row_decorator is not None: - def execute(instance, row, identitykey, isnew): + def execute(instance, row, flags): if self in selectcontext.recursion_stack: return decorated_row = row_decorator(row) @@ -639,7 +633,7 @@ class EagerLoader(AbstractRelationLoader): if not self.uselist: if self._should_log_debug: self.logger.debug("eagerload scalar instance on %s" % mapperutil.attribute_str(instance, self.key)) - if isnew: + if flags['isnew']: # set a scalar object instance directly on the parent object, # bypassing InstrumentedAttribute event handlers. instance.__dict__[self.key] = self.mapper._instance(selectcontext, decorated_row, None) @@ -648,7 +642,7 @@ class EagerLoader(AbstractRelationLoader): # so that we further descend into properties self.mapper._instance(selectcontext, decorated_row, None) else: - if isnew: + if flags['isnew']: if self._should_log_debug: self.logger.debug("initialize UniqueAppender on %s" % mapperutil.attribute_str(instance, self.key)) diff --git a/test/orm/mapper.py b/test/orm/mapper.py index 4cb721eeb5..9f12a4d1a9 100644 --- a/test/orm/mapper.py +++ b/test/orm/mapper.py @@ -574,7 +574,7 @@ class MapperTest(MapperSuperTest): def testextensionoptions(self): sess = create_session() class ext1(MapperExtension): - def populate_instance(self, mapper, selectcontext, row, instance, identitykey, isnew): + def populate_instance(self, mapper, selectcontext, row, instance, flags): """test options at the Mapper._instance level""" instance.TEST = "hello world" return EXT_PASS @@ -585,7 +585,7 @@ class MapperTest(MapperSuperTest): def select_by(self, *args, **kwargs): """test options at the Query level""" return "HI" - def populate_instance(self, mapper, selectcontext, row, instance, identitykey, isnew): + def populate_instance(self, mapper, selectcontext, row, instance, flags): """test options at the Mapper._instance level""" instance.TEST_2 = "also hello world" return EXT_PASS diff --git a/test/perf/masseagerload.py b/test/perf/masseagerload.py index 8563dab1af..3e865a0414 100644 --- a/test/perf/masseagerload.py +++ b/test/perf/masseagerload.py @@ -8,7 +8,8 @@ import time db = testbase.db -NUM = 25000 +NUM = 500 +DIVISOR = 50 class LoadTest(AssertMixin): def setUpAll(self): @@ -27,15 +28,16 @@ class LoadTest(AssertMixin): def setUp(self): clear_mappers() l = [] - for x in range(1,NUM/500): + for x in range(1,NUM/DIVISOR): l.append({'item_id':x, 'value':'this is item #%d' % x}) + print l items.insert().execute(*l) - for x in range(1, NUM/500): + for x in range(1, NUM/DIVISOR): l = [] - for y in range(1, NUM/(NUM/500)): - z = ((x-1) * NUM/(NUM/500)) + y + for y in range(1, NUM/(NUM/DIVISOR)): + z = ((x-1) * NUM/(NUM/DIVISOR)) + y l.append({'sub_id':z,'value':'this is iteim #%d' % z, 'parent_id':x}) - #print l + print l subitems.insert().execute(*l) def testload(self): class Item(object):pass