From: Mike Bayer Date: Thu, 31 May 2007 22:04:21 +0000 (+0000) Subject: - added undefer_group() MapperOption, sets a set of "deferred" columns joined by a X-Git-Tag: rel_0_4_6~229 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2ecb30b561d552d9d9fa34c46fb7ab29ae99153e;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - added undefer_group() MapperOption, sets a set of "deferred" columns joined by a "group" to load as "undeferred". --- diff --git a/CHANGES b/CHANGES index d6799da84e..32d7103c63 100644 --- a/CHANGES +++ b/CHANGES @@ -1,11 +1,19 @@ 0.4.0 - - orm - - deferred inheritance loading: polymorphic mappers can be constructed *without* + - along with recent speedups to ResultProxy, total number of function calls + significantly reduced for large loads. test/perf/masseagerload.py reports + 0.4 as having the fewest number of function calls across all SA versions + (0.1, 0.2, and 0.3) + - secondary inheritance loading: polymorphic mappers can be constructed *without* a select_table argument. inheriting mappers whose tables were not represented in the initial load will issue a second SQL query immediately, once per instance (i.e. not very efficient for large lists), in order to load the remaining columns. + - secondary inheritance loading can also move its second query into a column- + level "deferred" load, via the "polymorphic_fetch" argument, which can be set + to 'select' or 'deferred' + - added undefer_group() MapperOption, sets a set of "deferred" columns joined by a + "group" to load as "undeferred". 0.3.XXX - engines diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index e9caa49965..ed9a5b228b 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -892,7 +892,7 @@ class ResultProxy(object): self.__props[i] = rec if self.__echo: - self.context.engine.logger.debug("Cls " + repr(tuple([x[0] for x in metadata]))) + self.context.engine.logger.debug("Col " + repr(tuple([x[0] for x in metadata]))) def close(self): """Close this ResultProxy, and the underlying DBAPI cursor corresponding to the execution. diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index ce22f46239..83a31eee24 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import properties, strategies, interfaces from sqlalchemy.orm.session import Session as create_session from sqlalchemy.orm.session import object_session, attribute_manager -__all__ = ['relation', 'column_property', 'backref', 'eagerload', 'lazyload', 'noload', 'deferred', 'defer', 'undefer', 'extension', +__all__ = ['relation', 'column_property', 'backref', 'eagerload', 'lazyload', 'noload', 'deferred', 'defer', 'undefer', 'undefer_group', 'extension', 'mapper', 'clear_mappers', 'compile_mappers', 'clear_mapper', 'class_mapper', 'object_mapper', 'MapperExtension', 'Query', 'cascade_mappers', 'polymorphic_union', 'create_session', 'synonym', 'contains_alias', 'contains_eager', 'EXT_PASS', 'object_session' ] @@ -241,7 +241,14 @@ def undefer(name): return strategies.DeferredOption(name, defer=False) +def undefer_group(name): + """Return a ``MapperOption`` that will convert the given + group of deferred column properties into a non-deferred (regular column) load. + Used with ``query.options()``. + """ + return strategies.UndeferGroupOption(name) + def cascade_mappers(*classes_or_mappers): """Attempt to create a series of ``relations()`` between mappers automatically, via introspecting the foreign key relationships of diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 4f93737bdc..c5da017322 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -161,13 +161,7 @@ class StrategizedProperty(MapperProperty): """ def _get_context_strategy(self, context): - try: - return context.attributes[id(self)] - except KeyError: - # cache the located strategy per StrategizedProperty in the given context for faster re-lookup - ctx_strategy = self._get_strategy(context.attributes.get((LoaderStrategy, self), self.strategy.__class__)) - context.attributes[id(self)] = ctx_strategy - return ctx_strategy + return self._get_strategy(context.attributes.get(("loaderstrategy", self), self.strategy.__class__)) def _get_strategy(self, cls): try: @@ -266,11 +260,11 @@ class StrategizedOption(PropertyOption): def process_query_property(self, context, property): self.logger.debug("applying option to QueryContext, property key '%s'" % self.key) - context.attributes[(LoaderStrategy, property)] = self.get_strategy_class() + context.attributes[("loaderstrategy", property)] = self.get_strategy_class() def process_selection_property(self, context, property): self.logger.debug("applying option to SelectionContext, property key '%s'" % self.key) - context.attributes[(LoaderStrategy, property)] = self.get_strategy_class() + context.attributes[("loaderstrategy", property)] = self.get_strategy_class() def get_strategy_class(self): raise NotImplementedError() diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index aff5705050..c24de84bf5 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -624,7 +624,7 @@ class Mapper(object): """ # object attribute names mapped to MapperProperty objects - self.__props = {} + self.__props = util.OrderedDict() # table columns mapped to lists of MapperProperty objects # using a list allows a single column to be defined as diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 1d592f4e59..6b35855989 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -95,7 +95,9 @@ class DeferredColumnLoader(LoaderStrategy): """ def create_row_processor(self, selectcontext, mapper, row): - if not self.is_default or len(selectcontext.options): + if self.group is not None and selectcontext.attributes.get(('undefer', self.group), False): + return self.parent_property._get_strategy(ColumnLoader).create_row_processor(selectcontext, mapper, row) + elif not self.is_default or len(selectcontext.options): def execute(instance, row, isnew, **flags): if isnew: if self._should_log_debug: @@ -121,7 +123,8 @@ class DeferredColumnLoader(LoaderStrategy): sessionlib.attribute_manager.register_attribute(self.parent.class_, self.key, uselist=False, callable_=lambda i:self.setup_loader(i), copy_function=lambda x: self.columns[0].type.copy_value(x), compare_function=lambda x,y:self.columns[0].type.compare_values(x,y), mutable_scalars=self.columns[0].type.is_mutable()) def setup_query(self, context, **kwargs): - pass + if self.group is not None and context.attributes.get(('undefer', self.group), False): + self.parent_property._get_strategy(ColumnLoader).setup_query(context, **kwargs) def setup_loader(self, instance): localparent = mapper.object_mapper(instance, raiseerror=False) @@ -185,6 +188,15 @@ class DeferredOption(StrategizedOption): else: return ColumnLoader +class UndeferGroupOption(MapperOption): + def __init__(self, group): + self.group = group + def process_query_context(self, context): + context.attributes[('undefer', self.group)] = True + + def process_selection_context(self, context): + context.attributes[('undefer', self.group)] = True + class AbstractRelationLoader(LoaderStrategy): def init(self): super(AbstractRelationLoader, self).init() @@ -690,6 +702,8 @@ class EagerLazyOption(StrategizedOption): EagerLazyOption.logger = logging.class_logger(EagerLazyOption) + + class FetchModeOption(PropertyOption): def __init__(self, key, type): super(FetchModeOption, self).__init__(key) diff --git a/test/orm/mapper.py b/test/orm/mapper.py index 8dd9c0cca3..0b8ca19910 100644 --- a/test/orm/mapper.py +++ b/test/orm/mapper.py @@ -914,6 +914,27 @@ class DeferredTest(MapperSuperTest): ("SELECT orders.order_id AS orders_order_id, orders.user_id AS orders_user_id, orders.description AS orders_description, orders.isopen AS orders_isopen FROM orders ORDER BY %s" % orderby, {}), ]) + def testundefergroup(self): + """tests undefer_group()""" + m = mapper(Order, orders, properties = { + 'userident':deferred(orders.c.user_id, group='primary'), + 'description':deferred(orders.c.description, group='primary'), + 'opened':deferred(orders.c.isopen, group='primary') + }) + sess = create_session() + q = sess.query(m) + def go(): + l = q.options(undefer_group('primary')).select() + o2 = l[2] + print o2.opened, o2.description, o2.userident + assert o2.opened == 1 + assert o2.userident == 7 + assert o2.description == 'order 3' + orderby = str(orders.default_order_by()[0].compile(db)) + self.assert_sql(db, go, [ + ("SELECT orders.user_id AS orders_user_id, orders.description AS orders_description, orders.isopen AS orders_isopen, orders.order_id AS orders_order_id FROM orders ORDER BY %s" % orderby, {}), + ]) + def testdeepoptions(self): m = mapper(User, users, properties={ diff --git a/test/testbase.py b/test/testbase.py index 9ee79da20d..fdbe6aa5b4 100644 --- a/test/testbase.py +++ b/test/testbase.py @@ -310,6 +310,7 @@ class ExecutionContextWrapper(object): testdata.buffer.write(statement + "\n") if testdata.assert_list is not None: + assert len(testdata.assert_list), "Received query but no more assertions: %s" % statement item = testdata.assert_list[-1] if not isinstance(item, dict): item = testdata.assert_list.pop() @@ -330,7 +331,7 @@ class ExecutionContextWrapper(object): testdata.assert_list.pop() item = (statement, entry) except KeyError: - self.unittest.assert_(False, "Testing for one of the following queries: %s, received '%s'" % (repr([k for k in item.keys()]), statement)) + assert False, "Testing for one of the following queries: %s, received '%s'" % (repr([k for k in item.keys()]), statement) (query, params) = item if callable(params): @@ -344,7 +345,7 @@ class ExecutionContextWrapper(object): parameters = [p.get_original_dict() for p in ctx.compiled_parameters] query = self.convert_statement(query) - testdata.unittest.assert_(statement == query and (params is None or params == parameters), "Testing for query '%s' params %s, received '%s' with params %s" % (query, repr(params), statement, repr(parameters))) + assert statement == query and (params is None or params == parameters), "Testing for query '%s' params %s, received '%s' with params %s" % (query, repr(params), statement, repr(parameters)) testdata.sql_count += 1 self.ctx.post_exec()