From: Mike Bayer Date: Wed, 7 Dec 2005 02:57:22 +0000 (+0000) Subject: limit and offset support for mappers, insanity with eager loading X-Git-Tag: rel_0_1_0~251 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c9bfe709611cc419b4ac44d0db258e57682ebf77;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git limit and offset support for mappers, insanity with eager loading --- diff --git a/doc/build/content/adv_datamapping.myt b/doc/build/content/adv_datamapping.myt index 306da3f818..29d0e7ec1c 100644 --- a/doc/build/content/adv_datamapping.myt +++ b/doc/build/content/adv_datamapping.myt @@ -3,7 +3,7 @@ <&|doclib.myt:item, name="adv_datamapping", description="Advanced Data Mapping" &>

This section is under construction. For now, it has just the basic recipe for each concept without much else.

-

To start, heres the tables we will work with again:

+

To start, heres the tables w e will work with again:

<&|formatting.myt:code&> from sqlalchemy import * db = create_engine('sqlite://filename=mydb', echo=True) @@ -131,7 +131,47 @@ - +<&|doclib.myt:item, name="limits", description="Limiting Rows" &> +

You can limit rows in a regular SQL query by specifying limit and offset. A Mapper can handle the same concepts:

+<&|formatting.myt:code&> + class User(object): + pass + + m = mapper(User, users) +<&formatting.myt:poplink&>r = m.select(limit=20, offset=10) +<&|formatting.myt:codepopper, link="sql" &>SELECT users.user_id AS users_user_id, +users.user_name AS users_user_name, users.password AS users_password +FROM users ORDER BY users.oid + LIMIT 20 OFFSET 10 +{} + + +However, things get very tricky when dealing with eager relationships, since a straight LIMIT is not accurate with regards to child items. So here is what SQLAlchemy will do when you use limit or offset with an eager relationship: + <&|formatting.myt:code&> + class User(object): + pass + class Address(object): + pass + m = mapper(User, users, properties={ + 'addresses' : relation(Address, addresses, lazy=False) + }) + r = m.select(limit=20, offset=10) +<&|formatting.myt:poppedcode, link="sql" &> +SELECT users.user_id AS users_user_id, users.user_name AS users_user_name, +users.password AS users_password, addresses.address_id AS addresses_address_id, +addresses.user_id AS addresses_user_id, addresses.street AS addresses_street, +addresses.city AS addresses_city, addresses.state AS addresses_state, +addresses.zip AS addresses_zip +FROM +(SELECT users.user_id FROM users ORDER BY users.oid LIMIT 20 OFFSET 10) AS rowcount, + users LEFT OUTER JOIN addresses ON users.user_id = addresses.user_id +WHERE rowcount.user_id = users.user_id ORDER BY addresses.oid +{} + + + +

A subquery is used to create the limited set of rows, which is then joined to the larger eager query.

+ <&|doclib.myt:item, name="options", description="Mapper Options" &>

The options method of mapper produces a copy of the mapper, with modified properties and/or options. This makes it easy to take a mapper and just change a few things on it. The method takes a variable number of MapperOption objects which know how to change specific things about the mapper. The four available options are eagerload, lazyload, noload and extension.

An example of a mapper with a lazy load relationship, upgraded to an eager load relationship: diff --git a/doc/build/content/sqlconstruction.myt b/doc/build/content/sqlconstruction.myt index 21bc5799aa..4453286663 100644 --- a/doc/build/content/sqlconstruction.myt +++ b/doc/build/content/sqlconstruction.myt @@ -313,6 +313,20 @@ ORDER BY users.user_id DESC, users.user_name ASC + <&|doclib.myt:item, name="options", description="DISTINCT, LIMIT and OFFSET" &> + These are specified as keyword arguments: + <&|formatting.myt:code &> + <&formatting.myt:poplink&>c = select([users.c.user_name], distinct=True).execute() +<&|formatting.myt:codepopper, link="sql" &> +SELECT DISTINCT users.user_name FROM users + + <&formatting.myt:poplink&>c = users.select(limit=10, offset=20).execute() +<&|formatting.myt:codepopper, link="sql" &> +SELECT users.user_id, users.user_name, users.password FROM users LIMIT 10 OFFSET 20 + + + + The Oracle driver does not support LIMIT and OFFSET directly, but instead wraps the generated query into a subquery and uses the "rownum" variable to control the rows selected (this is somewhat experimental). <&|doclib.myt:item, name="join", description="Inner and Outer Joins" &> diff --git a/lib/sqlalchemy/mapping/mapper.py b/lib/sqlalchemy/mapping/mapper.py index 57e7887e99..092a0c0d53 100644 --- a/lib/sqlalchemy/mapping/mapper.py +++ b/lib/sqlalchemy/mapping/mapper.py @@ -71,7 +71,7 @@ class Mapper(object): table = sql.join(table, inherits.table, inherit_condition) self.table = table - + # locate all tables contained within the "table" passed in, which # may be a join or other construct tf = TableFinder() @@ -158,6 +158,7 @@ class Mapper(object): if not hasattr(self.class_, '_mapper') or self.is_primary or not mapper_registry.has_key(self.class_._mapper) or (inherits is not None and inherits._is_primary_mapper()): self._init_class() + engines = property(lambda s: [t.engine for t in s.tables]) @@ -208,7 +209,10 @@ class Mapper(object): prop.init(key, self) - def instances(self, cursor, *mappers): + def instances(self, cursor, *mappers, **kwargs): + limit = kwargs.get('limit', None) + offset = kwargs.get('offset', None) + result = util.HistoryArraySet() if len(mappers): otherresults = [] @@ -292,7 +296,7 @@ class Mapper(object): else: return None - def select(self, arg = None, **params): + def select(self, arg = None, **kwargs): """selects instances of the object from the database. arg can be any ClauseElement, which will form the criterion with which to @@ -303,13 +307,16 @@ class Mapper(object): in this case, the developer must insure that an adequate set of columns exists in the rowset with which to build new object instances.""" if arg is not None and isinstance(arg, sql.Select): - return self.select_statement(arg, **params) + return self.select_statement(arg, **kwargs) else: - return self.select_whereclause(arg, **params) + return self.select_whereclause(arg, **kwargs) - def select_whereclause(self, whereclause = None, order_by = None, **params): - statement = self._compile(whereclause, order_by = order_by) - return self.select_statement(statement, **params) + def select_whereclause(self, whereclause = None, params=None, **kwargs): + statement = self._compile(whereclause, **kwargs) + if params is not None: + return self.select_statement(statement, **params) + else: + return self.select_statement(statement) def select_statement(self, statement, **params): statement.use_labels = True @@ -438,12 +445,23 @@ class Mapper(object): for prop in self.props.values(): prop.register_deleted(obj, uow) - def _compile(self, whereclause = None, order_by = None, **options): - statement = sql.select([self.table], whereclause, order_by = order_by) - statement.order_by(self.table.rowid_column) + def _compile(self, whereclause = None, **kwargs): + if getattr(self, '_has_eager', False) and (kwargs.has_key('limit') or kwargs.has_key('offset')): + s2 = sql.select(self.table.primary_key, whereclause, **kwargs) + s2.order_by(self.table.rowid_column) + s3 = s2.alias('rowcount') + crit = [] + for i in range(0, len(self.table.primary_key)): + crit.append(s3.primary_key[i] == self.table.primary_key[i]) + statement = sql.select([self.table], sql.and_(*crit)) + if kwargs.has_key('order_by'): + statement.order_by(kwargs['order_by']) + else: + statement = sql.select([self.table], whereclause, **kwargs) + statement.order_by(self.table.rowid_column) # plugin point for key, value in self.props.iteritems(): - value.setup(key, statement, **options) + value.setup(key, statement, **kwargs) statement.use_labels = True return statement diff --git a/lib/sqlalchemy/mapping/properties.py b/lib/sqlalchemy/mapping/properties.py index 3429555d3b..4b82157d33 100644 --- a/lib/sqlalchemy/mapping/properties.py +++ b/lib/sqlalchemy/mapping/properties.py @@ -474,7 +474,7 @@ class LazyLoader(PropertyLoader): order_by = [self.secondary.rowid_column] else: order_by = [] - result = self.mapper.select(self.lazywhere, order_by=order_by,**params) + result = self.mapper.select(self.lazywhere, order_by=order_by, params=params) else: result = [] if self.uselist: @@ -530,6 +530,7 @@ class EagerLoader(PropertyLoader): def init(self, key, parent): PropertyLoader.init(self, key, parent) + parent._has_eager = True # figure out tables in the various join clauses we have, because user-defined # whereclauses that reference the same tables will be converted to use # aliases of those tables @@ -656,7 +657,6 @@ class EagerLazyOption(MapperOption): return "EagerLazyOption(%s, %s)" % (repr(self.key), repr(self.toeager)) def process(self, mapper): - tup = self.key.split('.', 1) key = tup[0] oldprop = mapper.props[key] @@ -674,13 +674,8 @@ class EagerLazyOption(MapperOption): else: class_ = LazyLoader - self.kwargs.setdefault('primaryjoin', oldprop.primaryjoin) - self.kwargs.setdefault('secondaryjoin', oldprop.secondaryjoin) - self.kwargs.setdefault('foreignkey', oldprop.foreignkey) - self.kwargs.setdefault('uselist', oldprop.uselist) - self.kwargs.setdefault('private', oldprop.private) - self.kwargs.setdefault('live', oldprop.live) - self.kwargs.setdefault('selectalias', oldprop.selectalias) + for arg in ('primaryjoin', 'secondaryjoin', 'foreignkey', 'uselist', 'private', 'live', 'isoption', 'association', 'selectalias', 'order_by', 'attributeext'): + self.kwargs.setdefault(arg, getattr(oldprop, arg)) self.kwargs['isoption'] = True mapper.set_property(key, class_(submapper, oldprop.secondary, **self.kwargs )) diff --git a/test/mapper.py b/test/mapper.py index 81836f34b9..57ede7dbda 100644 --- a/test/mapper.py +++ b/test/mapper.py @@ -249,7 +249,7 @@ class EagerTest(MapperSuperTest): m = mapper(Address, addresses) m = mapper(User, users, properties = dict( - addresses = relation(m, lazy = False, selectalias='lala', order_by=[desc(addresses.c.email_address)]), + addresses = relation(m, lazy = False, order_by=[desc(addresses.c.email_address)]), )) l = m.select()