From: Mike Bayer Date: Sat, 30 Sep 2006 04:40:15 +0000 (+0000) Subject: - internal refactoring to mapper instances() method to use a X-Git-Tag: rel_0_3_0~102 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d117a15020aedabdd7726a0e4bd9018e44717dbd;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - internal refactoring to mapper instances() method to use a SelectionContext object to track state during the operation. SLIGHT API BREAKAGE: the append_result() and populate_instances() methods on MapperExtension have a slightly different method signature now as a result of the change; hoping that these methods are not in widespread use as of yet. --- diff --git a/CHANGES b/CHANGES index 8e1617c4ef..1ea658c719 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,12 @@ instance-level logging under "sqlalchemy...". Test suite includes "--log-info" and "--log-debug" arguments which work independently of --verbose/--quiet. Logging added to orm to allow tracking of mapper configurations, row iteration. +- internal refactoring to mapper instances() method to use a +SelectionContext object to track state during the operation. +SLIGHT API BREAKAGE: the append_result() and populate_instances() +methods on MapperExtension have a slightly different method signature +now as a result of the change; hoping that these methods are not +in widespread use as of yet. - updates to MS-SQL driver: -- fixes bug 261 (table reflection broken for MS-SQL case-sensitive databases) diff --git a/doc/build/compile_docstrings.py b/doc/build/compile_docstrings.py index 119dbd85bc..528da388a8 100644 --- a/doc/build/compile_docstrings.py +++ b/doc/build/compile_docstrings.py @@ -25,7 +25,7 @@ make_doc(obj=sql, classes=[sql.Engine, sql.AbstractDialect, sql.ClauseParameters make_doc(obj=schema) make_doc(obj=engine, classes=[engine.Connectable, engine.ComposedSQLEngine, engine.Connection, engine.Transaction, engine.Dialect, engine.ConnectionProvider, engine.ExecutionContext, engine.ResultProxy, engine.RowProxy]) make_doc(obj=strategies) -make_doc(obj=orm, classes=[orm.Mapper, orm.MapperExtension]) +make_doc(obj=orm, classes=[orm.Mapper, orm.MapperExtension, orm.SelectionContext]) make_doc(obj=orm.query, classes=[orm.query.Query]) make_doc(obj=orm.session, classes=[orm.session.Session, orm.session.SessionTransaction]) make_doc(obj=pool, classes=[pool.DBProxy, pool.Pool, pool.QueuePool, pool.SingletonThreadPool]) diff --git a/examples/adjacencytree/byroot_tree.py b/examples/adjacencytree/byroot_tree.py index d191710514..48793b936e 100644 --- a/examples/adjacencytree/byroot_tree.py +++ b/examples/adjacencytree/byroot_tree.py @@ -91,7 +91,8 @@ class TreeLoader(MapperExtension): if instance.root is instance: connection.execute(mapper.mapped_table.update(TreeNode.c.id==instance.id, values=dict(root_node_id=instance.id))) instance.root_id = instance.id - def append_result(self, mapper, session, row, imap, result, instance, isnew, populate_existing=False): + + def append_result(self, mapper, selectcontext, row, instance, identitykey, result, isnew): """runs as results from a SELECT statement are processed, and newly created or already-existing instances that correspond to each row are appended to result lists. This method will only append root nodes to the result list, and will attach child nodes to their appropriate parent @@ -101,8 +102,8 @@ class TreeLoader(MapperExtension): result.append(instance) else: if isnew or populate_existing: - parentnode = imap[mapper.identity_key(instance.parent_id)] - parentnode.children.append(instance, _mapper_nohistory=True) + parentnode = selectcontext.identity_map[mapper.identity_key(instance.parent_id)] + parentnode.children.append_without_event(instance) return False class TreeData(object): diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index e7dff0f0ff..fd912e617f 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -12,7 +12,7 @@ import query as querylib import session as sessionlib import weakref -__all__ = ['Mapper', 'MapperExtension', 'class_mapper', 'object_mapper', 'EXT_PASS'] +__all__ = ['Mapper', 'MapperExtension', 'class_mapper', 'object_mapper', 'EXT_PASS', 'SelectionContext'] # a dictionary mapping classes to their primary mappers mapper_registry = weakref.WeakKeyDictionary() @@ -27,7 +27,6 @@ NO_ATTRIBUTE = object() # returned by a MapperExtension method to indicate a "do nothing" response EXT_PASS = object() - class Mapper(object): """Persists object instances to and from schema.Table objects via the sql package. @@ -626,10 +625,8 @@ class Mapper(object): corresponding to the rows in the cursor.""" self.__log_debug("instances()") self.compile() - limit = kwargs.get('limit', None) - offset = kwargs.get('offset', None) - populate_existing = kwargs.get('populate_existing', False) - version_check = kwargs.get('version_check', False) + + context = SelectionContext(self, session, **kwargs) result = util.UniqueAppender([]) if mappers: @@ -637,23 +634,18 @@ class Mapper(object): for m in mappers: otherresults.append(util.UniqueAppender([])) - imap = {} - scratch = {} - imap['_scratch'] = scratch while True: row = cursor.fetchone() if row is None: break - self._instance(session, row, imap, result, populate_existing=populate_existing, version_check=version_check) + self._instance(context, row, result) i = 0 for m in mappers: - m._instance(session, row, imap, otherresults[i]) + m._instance(context, row, otherresults[i]) i+=1 # store new stuff in the identity map - for value in imap.values(): - if value is scratch: - continue + for value in context.identity_map.values(): session._register_persistent(value) if mappers: @@ -687,6 +679,8 @@ class Mapper(object): def options(self, *options, **kwargs): """uses this mapper as a prototype for a new mapper with different behavior. *options is a list of options directives, which include eagerload(), lazyload(), and noload()""" + # TODO: this whole options() scheme is going to change, and not rely upon + # making huge chains of copies anymore. stay tuned ! self.compile() optkey = repr([hash_key(o) for o in options]) try: @@ -1017,7 +1011,7 @@ class Mapper(object): def get_select_mapper(self): return self.__surrogate_mapper or self - def _instance(self, session, row, imap, result = None, populate_existing = False, version_check=False): + def _instance(self, context, row, result = None): """pulls an object instance from the given row and appends it to the given result list. if the instance already exists in the given identity map, its not added. in either case, executes all the property loaders on the instance to also process extra @@ -1028,33 +1022,33 @@ class Mapper(object): mapper = self.polymorphic_map[discriminator] if mapper is not self: row = self.translate_row(mapper, row) - return mapper._instance(session, row, imap, result=result, populate_existing=populate_existing) + return mapper._instance(context, row, result=result) # look in main identity map. if its there, we dont do anything to it, # including modifying any of its related items lists, as its already # been exposed to being modified by the application. - populate_existing = populate_existing or self.always_refresh + populate_existing = context.populate_existing or self.always_refresh identitykey = self._row_identity_key(row) - if session.has_key(identitykey): - instance = session._get(identitykey) + if context.session.has_key(identitykey): + instance = context.session._get(identitykey) self.__log_debug("_instance(): using existing instance %s identity %s" % (mapperutil.instance_str(instance), str(identitykey))) isnew = False - if version_check and self.version_id_col is not None and self._getattrbycolumn(instance, self.version_id_col) != row[self.version_id_col]: + if context.version_check and self.version_id_col is not None and self._getattrbycolumn(instance, self.version_id_col) != row[self.version_id_col]: raise exceptions.ConcurrentModificationError("Instance '%s' version of %s does not match %s" % (instance, self._getattrbycolumn(instance, self.version_id_col), row[self.version_id_col])) - if populate_existing or session.is_expired(instance, unexpire=True): - if not imap.has_key(identitykey): - imap[identitykey] = instance + if populate_existing or context.session.is_expired(instance, unexpire=True): + if not context.identity_map.has_key(identitykey): + context.identity_map[identitykey] = instance for prop in self.__props.values(): - prop.execute(session, instance, row, identitykey, imap, True) - if self.extension.append_result(self, session, row, imap, result, instance, isnew, populate_existing=populate_existing) is EXT_PASS: + prop.execute(context, instance, row, identitykey, True) + if self.extension.append_result(self, context, row, instance, identitykey, result, isnew) is EXT_PASS: if result is not None: result.append(instance) return instance # look in result-local identitymap for it. - exists = imap.has_key(identitykey) + exists = context.identity_map.has_key(identitykey) if not exists: if self.allow_null_pks: # check if *all* primary key cols in the result are None - this indicates @@ -1072,23 +1066,21 @@ class Mapper(object): return None # plugin point - instance = self.extension.create_instance(self, session, row, imap, self.class_) + instance = self.extension.create_instance(self, context, row, self.class_) if instance is EXT_PASS: - instance = self._create_instance(session) + instance = self._create_instance(context.session) self.__log_debug("_instance(): created new instance %s identity %s" % (mapperutil.instance_str(instance), str(identitykey))) - imap[identitykey] = instance + context.identity_map[identitykey] = instance isnew = True else: - instance = imap[identitykey] + instance = context.identity_map[identitykey] isnew = False - # plugin point - # call further mapper properties on the row, to pull further # instances from the row and possibly populate this item. - if self.extension.populate_instance(self, session, instance, row, identitykey, imap, isnew) is EXT_PASS: - self.populate_instance(session, instance, row, identitykey, imap, isnew) - if self.extension.append_result(self, session, row, imap, result, instance, isnew, populate_existing=populate_existing) is EXT_PASS: + if self.extension.populate_instance(self, context, row, instance, identitykey, isnew) is EXT_PASS: + self.populate_instance(context, instance, row, identitykey, isnew) + if self.extension.append_result(self, context, row, instance, identitykey, result, isnew) is EXT_PASS: if result is not None: result.append(instance) return instance @@ -1113,11 +1105,11 @@ class Mapper(object): newrow[c] = row[c2] return newrow - def populate_instance(self, session, instance, row, identitykey, imap, isnew, frommapper=None): + def populate_instance(self, selectcontext, instance, row, identitykey, isnew, frommapper=None): if frommapper is not None: row = frommapper.translate_row(self, row) for prop in self.__props.values(): - prop.execute(session, instance, row, identitykey, imap, isnew) + prop.execute(selectcontext, instance, row, identitykey, isnew) # deprecated query methods. Query is constructed from Session, and the rest # of these methods are called off of Query now. @@ -1180,11 +1172,47 @@ class Mapper(object): return self.query().select_text(text, **params) Mapper.logger = logging.class_logger(Mapper) + +class SelectionContext(object): + """created within the mapper.instances() method to store and share + state among all the Mappers and MapperProperty objects used in a load operation. + + SelectionContext contains these attributes: + + mapper - the Mapper which originated the instances() call. + + session - the Session that is relevant to the instances call. + + identity_map - a dictionary which stores newly created instances that have + not yet been added as persistent to the Session. + + attributes - a dictionary to store arbitrary data; eager loaders use it to + store additional result lists + + populate_existing - indicates if its OK to overwrite the attributes of instances + that were already in the Session + + version_check - indicates if mappers that have version_id columns should verify + that instances existing already within the Session should have this attribute compared + to the freshly loaded value + + """ + def __init__(self, mapper, session, **kwargs): + self.mapper = mapper + self.populate_existing = kwargs.get('populate_existing', False) + self.version_check = kwargs.get('version_check', False) + self.session = session + self.identity_map = {} + self.attributes = {} + class MapperProperty(object): """an element attached to a Mapper that describes and assists in the loading and saving of an attribute on an object instance.""" - def execute(self, session, instance, row, identitykey, imap, isnew): + def setup(self, statement, **options): + """called when a statement is being constructed. """ + return self + def execute(self, selectcontext, instance, row, identitykey, isnew): """called when the mapper receives a row. instance is the parent instance corresponding to the row. """ raise NotImplementedError() @@ -1202,9 +1230,6 @@ class MapperProperty(object): this is called by a mappers select_by method to formulate a set of key/value pairs into a WHERE criterion that spans multiple tables if needed.""" return None - def setup(self, key, statement, **options): - """called when a statement is being constructed. """ - return self def set_parent(self, parent): self.parent = parent def init(self, key, parent): @@ -1271,57 +1296,53 @@ class MapperExtension(object): def select(self, query, *args, **kwargs): """overrides the select method of the Query object""" return EXT_PASS - def create_instance(self, mapper, session, row, imap, class_): + def create_instance(self, mapper, selectcontext, row, class_): """called when a new object instance is about to be created from a row. the method can choose to create the instance itself, or it can return None to indicate normal object creation should take place. mapper - the mapper doing the operation + + selectcontext - SelectionContext corresponding to the instances() call row - the result row from the database - imap - a dictionary that is storing the running set of objects collected from the - current result set - class_ - the class we are mapping. """ return EXT_PASS - def append_result(self, mapper, session, row, imap, result, instance, isnew, populate_existing=False): + def append_result(self, mapper, selectcontext, row, instance, identitykey, result, isnew): """called when an object instance is being appended to a result list. - If this method returns True, it is assumed that the mapper should do the appending, else - if this method returns False, it is assumed that the append was handled by this method. + If this method returns EXT_PASS, it is assumed that the mapper should do the appending, else + if this method returns any other value or None, it is assumed that the append was handled by this method. mapper - the mapper doing the operation - row - the result row from the database - - imap - a dictionary that is storing the running set of objects collected from the - current result set + selectcontext - SelectionContext corresponding to the instances() call - result - an instance of util.HistoryArraySet(), which may be an attribute on an - object if this is a related object load (lazy or eager). + row - the result row from the database instance - the object instance to be appended to the result + identitykey - the identity key of the instance + + result - list to which results are being appended + isnew - indicates if this is the first time we have seen this object instance in the current result set. if you are selecting from a join, such as an eager load, you might see the same object instance many times in the same result set. - - populate_existing - usually False, indicates if object instances that were already in the main - identity map, i.e. were loaded by a previous select(), get their attributes overwritten """ return EXT_PASS - def populate_instance(self, mapper, session, instance, row, identitykey, imap, isnew): + def populate_instance(self, mapper, selectcontext, row, instance, identitykey, isnew): """called right before the mapper, after creating an instance from a row, passes the row to its MapperProperty objects which are responsible for populating the object's attributes. - If this method returns True, it is assumed that the mapper should do the appending, else - if this method returns False, it is assumed that the append was handled by this method. + If this method returns EXT_PASS, it is assumed that the mapper should do the appending, else + if this method returns any other value or None, it is assumed that the append was handled by this method. Essentially, this method is used to have a different mapper populate the object: - def populate_instance(self, mapper, session, instance, row, identitykey, imap, isnew): - othermapper.populate_instance(session, instance, row, identitykey, imap, isnew, frommapper=mapper) + def populate_instance(self, mapper, selectcontext, instance, row, identitykey, isnew): + othermapper.populate_instance(selectcontext, instance, row, identitykey, isnew, frommapper=mapper) return True """ return EXT_PASS diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index f76f46036a..3a7a7e6bae 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -32,7 +32,7 @@ class ColumnProperty(mapper.MapperProperty): return sessionlib.attribute_manager.get_history(obj, self.key, passive=passive) def copy(self): return ColumnProperty(*self.columns) - def setup(self, key, statement, eagertable=None, **options): + def setup(self, statement, eagertable=None, **options): for c in self.columns: if eagertable is not None: statement.append_column(eagertable.corresponding_column(c)) @@ -43,7 +43,7 @@ class ColumnProperty(mapper.MapperProperty): if self.is_primary(): self.logger.info("register managed attribute %s on class %s" % (self.key, self.parent.class_.__name__)) sessionlib.attribute_manager.register_attribute(self.parent.class_, self.key, uselist=False, copy_function=lambda x: self.columns[0].type.copy_value(x), compare_function=lambda x,y:self.columns[0].type.compare_values(x,y), mutable_scalars=self.columns[0].type.is_mutable()) - def execute(self, session, instance, row, identitykey, imap, isnew): + def execute(self, selectcontext, instance, row, identitykey, isnew): if isnew: self.logger.debug("populating %s with %s/%s" % (mapperutil.attribute_str(instance, self.key), row.__class__.__name__, self.columns[0].key)) # set a scalar object instance directly on the object, @@ -112,9 +112,9 @@ class DeferredColumnProperty(ColumnProperty): return session.scalar(self.localparent, sql.select([self.columns[0]], clause, use_labels=True),None) return lazyload - def setup(self, key, statement, **options): + def setup(self, statement, **options): pass - def execute(self, session, instance, row, identitykey, imap, isnew): + def execute(self, selectcontext, instance, row, identitykey, isnew): if isnew: if not self.is_primary(): sessionlib.attribute_manager.init_instance_attribute(instance, self.key, False, callable_=self.setup_loader(instance)) @@ -348,7 +348,7 @@ class PropertyLoader(mapper.MapperProperty): else: return self.primaryjoin - def execute(self, session, instance, row, identitykey, imap, isnew): + def execute(self, selectcontext, instance, row, identitykey, isnew): if self.is_primary(): return #print "PLAIN PROPLOADER EXEC NON-PRIAMRY", repr(id(self)), repr(self.mapper.class_), self.key @@ -437,7 +437,7 @@ class LazyLoader(PropertyLoader): return None return lazyload - def execute(self, session, instance, row, identitykey, imap, isnew): + def execute(self, selectcontext, instance, row, identitykey, isnew): if isnew: # new object instance being loaded from a result row if not self.is_primary(): @@ -577,7 +577,7 @@ class EagerLoader(LazyLoader): orderby[i].accept_visitor(self.aliasizer) return orderby - def setup(self, key, statement, eagertable=None, **options): + def setup(self, statement, eagertable=None, **options): """add a left outer join to the statement thats being constructed""" # initialize the "eager" chain of EagerLoader objects @@ -618,11 +618,10 @@ class EagerLoader(LazyLoader): self._aliasize_orderby(statement.order_by_clause, False) statement.append_from(statement._outerjoin) - for key, value in self.eagermapper.props.iteritems(): - value.setup(key, statement, eagertable=self.eagertarget) + for value in self.eagermapper.props.values(): + value.setup(statement, eagertable=self.eagertarget) - - def execute(self, session, instance, row, identitykey, imap, isnew): + def execute(self, selectcontext, instance, row, identitykey, isnew): """receive a row. tell our mapper to look for a new object instance in the row, and attach it to a list on the parent instance.""" @@ -633,7 +632,7 @@ class EagerLoader(LazyLoader): except KeyError: # else degrade to a lazy loader self.logger.debug("degrade to lazy loader on %s" % mapperutil.attribute_str(instance, self.key)) - LazyLoader.execute(self, session, instance, row, identitykey, imap, isnew) + LazyLoader.execute(self, selectcontext, instance, row, identitykey, isnew) return @@ -642,11 +641,11 @@ class EagerLoader(LazyLoader): if isnew: # set a scalar object instance directly on the parent object, # bypassing SmartProperty event handlers. - instance.__dict__[self.key] = self.eagermapper._instance(session, decorated_row, imap, None) + instance.__dict__[self.key] = self.eagermapper._instance(selectcontext, decorated_row, None) else: # call _instance on the row, even though the object has been created, # so that we further descend into properties - self.eagermapper._instance(session, decorated_row, imap, None) + self.eagermapper._instance(selectcontext, decorated_row, None) else: if isnew: self.logger.debug("initialize UniqueAppender on %s" % mapperutil.attribute_str(instance, self.key)) @@ -657,10 +656,10 @@ class EagerLoader(LazyLoader): appender = util.UniqueAppender(l.data) # store it in the "scratch" area, which is local to this load operation. - imap['_scratch'][(instance, self.key)] = appender - result_list = imap['_scratch'][(instance, self.key)] + selectcontext.attributes[(instance, self.key)] = appender + result_list = selectcontext.attributes[(instance, self.key)] self.logger.debug("eagerload list instance on %s" % mapperutil.attribute_str(instance, self.key)) - self.eagermapper._instance(session, decorated_row, imap, result_list) + self.eagermapper._instance(selectcontext, decorated_row, result_list) def _create_decorator_row(self): class EagerRowAdapter(object): diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index a317435e8a..a91947ac35 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -388,8 +388,8 @@ class Query(object): # plugin point # give all the attached properties a chance to modify the query - for key, value in self.mapper.props.iteritems(): - value.setup(key, statement, **kwargs) + for value in self.mapper.props.values(): + value.setup(statement, **kwargs) return statement