--- /dev/null
+.. change::
+ :tags: bug, orm
+ :tickets: 4270
+
+ Fixed an issue that was both a performance regression in 1.2 as well as an
+ incorrect result regarding the "baked" lazy loader, involving the
+ generation of cache keys from the original :class:`.Query` object's loader
+ options. If the loader options were built up in a "branched" style using
+ common base elements for multiple options, the same options would be
+ rendered into the cache key repeatedly, causing both a performance issue as
+ well as generating the wrong cache key. This is fixed, along with a
+ performance improvement when such "branched" options are applied via
+ :meth:`.Query.options` to prevent the same option objects from being
+ applied repeatedly.
# most MapperOptions write to the '_attributes' dictionary,
# so copy that as well
self._attributes = self._attributes.copy()
+ if '_unbound_load_dedupes' not in self._attributes:
+ self._attributes['_unbound_load_dedupes'] = set()
opts = tuple(util.flatten_iterator(args))
self._with_options = self._with_options + opts
if conditional:
if key != "loader":
continue
- endpoint = obj._of_type or obj.path.path[-1]
- chopped = self._chop_path(loader_path, path)
-
- if (
- # means loader_path and path are unrelated,
- # this does not need to be part of a cache key
- chopped is None
- ) or (
- # means no additional path with loader_path + path
- # and the endpoint isn't using of_type so isn't modified into
- # an alias or other unsafe entity
- not chopped and not obj._of_type
- ):
- continue
-
- serialized_path = []
-
- for token in chopped:
- if isinstance(token, util.string_types):
- serialized_path.append(token)
- elif token.is_aliased_class:
- return False
- elif token.is_property:
- serialized_path.append(token.key)
- else:
- assert token.is_mapper
- serialized_path.append(token.class_)
-
- if not serialized_path or endpoint != serialized_path[-1]:
- if endpoint.is_mapper:
- serialized_path.append(endpoint.class_)
- elif endpoint.is_aliased_class:
- return False
-
- serialized.append(
- (
- tuple(serialized_path) +
- (obj.strategy or ()) +
- (tuple([
- (key, obj.local_opts[key])
- for key in sorted(obj.local_opts)
- ]) if obj.local_opts else ())
+ for local_elem, obj_elem in zip(self.path.path, loader_path):
+ if local_elem is not obj_elem:
+ break
+ else:
+ endpoint = obj._of_type or obj.path.path[-1]
+ chopped = self._chop_path(loader_path, path)
+
+ if (
+ # means loader_path and path are unrelated,
+ # this does not need to be part of a cache key
+ chopped is None
+ ) or (
+ # means no additional path with loader_path + path
+ # and the endpoint isn't using of_type so isn't modified
+ # into an alias or other unsafe entity
+ not chopped and not obj._of_type
+ ):
+ continue
+
+ serialized_path = []
+
+ for token in chopped:
+ if isinstance(token, util.string_types):
+ serialized_path.append(token)
+ elif token.is_aliased_class:
+ return False
+ elif token.is_property:
+ serialized_path.append(token.key)
+ else:
+ assert token.is_mapper
+ serialized_path.append(token.class_)
+
+ if not serialized_path or endpoint != serialized_path[-1]:
+ if endpoint.is_mapper:
+ serialized_path.append(endpoint.class_)
+ elif endpoint.is_aliased_class:
+ return False
+
+ serialized.append(
+ (
+ tuple(serialized_path) +
+ (obj.strategy or ()) +
+ (tuple([
+ (key, obj.local_opts[key])
+ for key in sorted(obj.local_opts)
+ ]) if obj.local_opts else ())
+ )
)
- )
if not serialized:
return None
else:
def _generate_cache_key(self, path):
serialized = ()
for val in self._to_bind:
- opt = val._bind_loader(
- [path.path[0]],
- None, None, False)
- if opt:
- c_key = opt._generate_cache_key(path)
- if c_key is False:
- return False
- elif c_key:
- serialized += c_key
+ for local_elem, val_elem in zip(self.path, val.path):
+ if local_elem is not val_elem:
+ break
+ else:
+ opt = val._bind_loader(
+ [path.path[0]],
+ None, None, False)
+ if opt:
+ c_key = opt._generate_cache_key(path)
+ if c_key is False:
+ return False
+ elif c_key:
+ serialized += c_key
if not serialized:
return None
else:
self.__dict__ = state
def _process(self, query, raiseerr):
+ dedupes = query._attributes['_unbound_load_dedupes']
for val in self._to_bind:
- val._bind_loader(
- [ent.entity_zero for ent in query._mapper_entities],
- query._current_path, query._attributes, raiseerr)
+ if val not in dedupes:
+ dedupes.add(val)
+ val._bind_loader(
+ [ent.entity_zero for ent in query._mapper_entities],
+ query._current_path, query._attributes, raiseerr)
@classmethod
def _from_keys(cls, meth, keys, chained, kw):
from sqlalchemy import Integer, String, ForeignKey
from sqlalchemy.orm import mapper, relationship, \
- sessionmaker, Session, defer, joinedload, defaultload, selectinload
+ sessionmaker, Session, defer, joinedload, defaultload, selectinload, \
+ Load, configure_mappers
from sqlalchemy import testing
from sqlalchemy.testing import profiling
from sqlalchemy.testing import fixtures
from sqlalchemy.testing.schema import Table, Column
-
+from sqlalchemy import inspect
class MergeTest(fixtures.MappedTest):
sess.close()
go()
+
class JoinedEagerLoadTest(fixtures.MappedTest):
@classmethod
def define_tables(cls, metadata):
list(obj)
sess.close()
go()
+
+
+class BranchedOptionTest(fixtures.MappedTest):
+ @classmethod
+ def define_tables(cls, metadata):
+ def make_some_columns():
+ return [
+ Column('c%d' % i, Integer)
+ for i in range(2)
+ ]
+
+ Table(
+ 'a',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ *make_some_columns()
+ )
+ Table(
+ 'b',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('a_id', ForeignKey('a.id')),
+ *make_some_columns()
+ )
+ Table(
+ 'c',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('b_id', ForeignKey('b.id')),
+ *make_some_columns()
+ )
+ Table(
+ 'd',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('b_id', ForeignKey('b.id')),
+ *make_some_columns()
+ )
+ Table(
+ 'e',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('b_id', ForeignKey('b.id')),
+ *make_some_columns()
+ )
+ Table(
+ 'f',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('b_id', ForeignKey('b.id')),
+ *make_some_columns()
+ )
+ Table(
+ 'g',
+ metadata,
+ Column('id', Integer,
+ primary_key=True, test_needs_autoincrement=True),
+ Column('a_id', ForeignKey('a.id')),
+ *make_some_columns()
+ )
+
+ @classmethod
+ def setup_classes(cls):
+ class A(cls.Basic):
+ pass
+
+ class B(cls.Basic):
+ pass
+
+ class C(cls.Basic):
+ pass
+
+ class D(cls.Basic):
+ pass
+
+ class E(cls.Basic):
+ pass
+
+ class F(cls.Basic):
+ pass
+
+ class G(cls.Basic):
+ pass
+
+ @classmethod
+ def setup_mappers(cls):
+ A, B, C, D, E, F, G = cls.classes('A', 'B', 'C', 'D', 'E', 'F', 'G')
+ a, b, c, d, e, f, g = cls.tables('a', 'b', 'c', 'd', 'e', 'f', 'g')
+
+ mapper(A, a, properties={
+ 'bs': relationship(B),
+ 'gs': relationship(G)
+ })
+ mapper(B, b, properties={
+ 'cs': relationship(C),
+ 'ds': relationship(D),
+ 'es': relationship(E),
+ 'fs': relationship(F)
+ })
+ mapper(C, c)
+ mapper(D, d)
+ mapper(E, e)
+ mapper(F, f)
+ mapper(G, g)
+
+ configure_mappers()
+
+ def test_generate_cache_key_unbound_branching(self):
+ A, B, C, D, E, F, G = self.classes('A', 'B', 'C', 'D', 'E', 'F', 'G')
+
+ base = joinedload(A.bs)
+ opts = [
+ base.joinedload(B.cs),
+ base.joinedload(B.ds),
+ base.joinedload(B.es),
+ base.joinedload(B.fs)
+ ]
+
+ cache_path = inspect(A)._path_registry
+
+ @profiling.function_call_count()
+ def go():
+ for opt in opts:
+ opt._generate_cache_key(cache_path)
+ go()
+
+ def test_generate_cache_key_bound_branching(self):
+ A, B, C, D, E, F, G = self.classes('A', 'B', 'C', 'D', 'E', 'F', 'G')
+
+ base = Load(A).joinedload(A.bs)
+ opts = [
+ base.joinedload(B.cs),
+ base.joinedload(B.ds),
+ base.joinedload(B.es),
+ base.joinedload(B.fs)
+ ]
+
+ cache_path = inspect(A)._path_registry
+
+ @profiling.function_call_count()
+ def go():
+ for opt in opts:
+ opt._generate_cache_key(cache_path)
+ go()
+
+ def test_query_opts_unbound_branching(self):
+ A, B, C, D, E, F, G = self.classes('A', 'B', 'C', 'D', 'E', 'F', 'G')
+
+ base = joinedload(A.bs)
+ opts = [
+ base.joinedload(B.cs),
+ base.joinedload(B.ds),
+ base.joinedload(B.es),
+ base.joinedload(B.fs)
+ ]
+
+ q = Session().query(A)
+
+ @profiling.function_call_count()
+ def go():
+ q.options(*opts)
+ go()
+
+ def test_query_opts_key_bound_branching(self):
+ A, B, C, D, E, F, G = self.classes('A', 'B', 'C', 'D', 'E', 'F', 'G')
+
+ base = Load(A).joinedload(A.bs)
+ opts = [
+ base.joinedload(B.cs),
+ base.joinedload(B.ds),
+ base.joinedload(B.es),
+ base.joinedload(B.fs)
+ ]
+
+ q = Session().query(A)
+
+ @profiling.function_call_count()
+ def go():
+ q.options(*opts)
+ go()
+
)
)
+ def test_unbound_cache_key_included_safe_multipath(self):
+ User, Address, Order, Item, SubItem = self.classes(
+ 'User', 'Address', 'Order', 'Item', 'SubItem')
+
+ query_path = self._make_path_registry([User, "orders"])
+
+ base = joinedload(User.orders)
+ opt1 = base.joinedload(Order.items)
+ opt2 = base.joinedload(Order.address)
+
+ eq_(
+ opt1._generate_cache_key(query_path),
+ (
+ ((Order, 'items', Item, ('lazy', 'joined')),)
+ )
+ )
+
+ eq_(
+ opt2._generate_cache_key(query_path),
+ (
+ ((Order, 'address', Address, ('lazy', 'joined')),)
+ )
+ )
+
+ def test_bound_cache_key_included_safe_multipath(self):
+ User, Address, Order, Item, SubItem = self.classes(
+ 'User', 'Address', 'Order', 'Item', 'SubItem')
+
+ query_path = self._make_path_registry([User, "orders"])
+
+ base = Load(User).joinedload(User.orders)
+ opt1 = base.joinedload(Order.items)
+ opt2 = base.joinedload(Order.address)
+
+ eq_(
+ opt1._generate_cache_key(query_path),
+ (
+ ((Order, 'items', Item, ('lazy', 'joined')),)
+ )
+ )
+
+ eq_(
+ opt2._generate_cache_key(query_path),
+ (
+ ((Order, 'address', Address, ('lazy', 'joined')),)
+ )
+ )
+
def test_bound_cache_key_included_safe(self):
User, Address, Order, Item, SubItem = self.classes(
'User', 'Address', 'Order', 'Item', 'SubItem')
)
)
+ def test_unbound_cache_key_included_safe_w_deferred_multipath(self):
+ User, Address, Order, Item, SubItem = self.classes(
+ 'User', 'Address', 'Order', 'Item', 'SubItem')
+
+ query_path = self._make_path_registry([User, "orders"])
+
+ base = joinedload(User.orders)
+ opt1 = base.joinedload(Order.items)
+ opt2 = base.joinedload(Order.address).defer(Address.email_address).\
+ defer(Address.user_id)
+
+ eq_(
+ opt1._generate_cache_key(query_path),
+ (
+ (Order, 'items', Item, ('lazy', 'joined')),
+ )
+ )
+
+ eq_(
+ opt2._generate_cache_key(query_path),
+ (
+ (Order, 'address', Address, ('lazy', 'joined')),
+ (Order, 'address', Address, 'email_address',
+ ('deferred', True), ('instrument', True)),
+ (Order, 'address', Address, 'user_id',
+ ('deferred', True), ('instrument', True))
+ )
+ )
+
def test_bound_cache_key_included_safe_w_deferred(self):
User, Address, Order, Item, SubItem = self.classes(
'User', 'Address', 'Order', 'Item', 'SubItem')
)
)
+ def test_bound_cache_key_included_safe_w_deferred_multipath(self):
+ User, Address, Order, Item, SubItem = self.classes(
+ 'User', 'Address', 'Order', 'Item', 'SubItem')
+
+ query_path = self._make_path_registry([User, "orders"])
+
+ base = Load(User).joinedload(User.orders)
+ opt1 = base.joinedload(Order.items)
+ opt2 = base.joinedload(Order.address).defer(Address.email_address).\
+ defer(Address.user_id)
+
+ eq_(
+ opt1._generate_cache_key(query_path),
+ (
+ (Order, 'items', Item, ('lazy', 'joined')),
+ )
+ )
+
+ eq_(
+ opt2._generate_cache_key(query_path),
+ (
+ (Order, 'address', Address, ('lazy', 'joined')),
+ (Order, 'address', Address, 'email_address',
+ ('deferred', True), ('instrument', True)),
+ (Order, 'address', Address, 'user_id',
+ ('deferred', True), ('instrument', True))
+ )
+ )
+
def test_unbound_cache_key_included_safe_w_option(self):
User, Address, Order, Item, SubItem = self.classes(
'User', 'Address', 'Order', 'Item', 'SubItem')
# /home/classic/dev/sqlalchemy/test/profiles.txt
# This file is written out on a per-environment basis.
-# For each test in aaa_profiling, the corresponding function and
+# For each test in aaa_profiling, the corresponding function and
# environment is located within this file. If it doesn't exist,
# the test is skipped.
-# If a callcount does exist, it is compared to what we received.
+# If a callcount does exist, it is compared to what we received.
# assertions are raised if the counts do not match.
-#
-# To add a new callcount test, apply the function_call_count
-# decorator and re-run the tests using the --write-profiles
+#
+# To add a new callcount test, apply the function_call_count
+# decorator and re-run the tests using the --write-profiles
# option - this file will be rewritten including the new count.
-#
+#
# TEST: test.aaa_profiling.test_compiler.CompileTest.test_insert
test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.6_sqlite_pysqlite_dbapiunicode_cextensions 150
test.aaa_profiling.test_compiler.CompileTest.test_update_whereclause 3.6_sqlite_pysqlite_dbapiunicode_nocextensions 150
+# TEST: test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching
+
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_mysql_mysqldb_dbapiunicode_cextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_mysql_mysqldb_dbapiunicode_nocextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_postgresql_psycopg2_dbapiunicode_cextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_sqlite_pysqlite_dbapiunicode_cextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 104
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_mysql_mysqldb_dbapiunicode_cextensions 81
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_mysql_mysqldb_dbapiunicode_nocextensions 81
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_postgresql_psycopg2_dbapiunicode_cextensions 81
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_postgresql_psycopg2_dbapiunicode_nocextensions 81
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_sqlite_pysqlite_dbapiunicode_cextensions 81
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_bound_branching 3.6_sqlite_pysqlite_dbapiunicode_nocextensions 81
+
+# TEST: test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching
+
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_mysql_mysqldb_dbapiunicode_cextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_mysql_mysqldb_dbapiunicode_nocextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_postgresql_psycopg2_dbapiunicode_cextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_sqlite_pysqlite_dbapiunicode_cextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 651
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_mysql_mysqldb_dbapiunicode_cextensions 624
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_mysql_mysqldb_dbapiunicode_nocextensions 624
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_postgresql_psycopg2_dbapiunicode_cextensions 624
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_postgresql_psycopg2_dbapiunicode_nocextensions 624
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_sqlite_pysqlite_dbapiunicode_cextensions 624
+test.aaa_profiling.test_orm.BranchedOptionTest.test_generate_cache_key_unbound_branching 3.6_sqlite_pysqlite_dbapiunicode_nocextensions 624
+
+# TEST: test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching
+
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_mysql_mysqldb_dbapiunicode_cextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_mysql_mysqldb_dbapiunicode_nocextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_postgresql_psycopg2_dbapiunicode_cextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_sqlite_pysqlite_dbapiunicode_cextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 45
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_mysql_mysqldb_dbapiunicode_cextensions 46
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_mysql_mysqldb_dbapiunicode_nocextensions 46
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_postgresql_psycopg2_dbapiunicode_cextensions 46
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_postgresql_psycopg2_dbapiunicode_nocextensions 46
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_sqlite_pysqlite_dbapiunicode_cextensions 46
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_key_bound_branching 3.6_sqlite_pysqlite_dbapiunicode_nocextensions 46
+
+# TEST: test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching
+
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_mysql_mysqldb_dbapiunicode_cextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_mysql_mysqldb_dbapiunicode_nocextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_postgresql_psycopg2_dbapiunicode_cextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_sqlite_pysqlite_dbapiunicode_cextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 460
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_mysql_mysqldb_dbapiunicode_cextensions 466
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_mysql_mysqldb_dbapiunicode_nocextensions 466
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_postgresql_psycopg2_dbapiunicode_cextensions 466
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_postgresql_psycopg2_dbapiunicode_nocextensions 466
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_sqlite_pysqlite_dbapiunicode_cextensions 466
+test.aaa_profiling.test_orm.BranchedOptionTest.test_query_opts_unbound_branching 3.6_sqlite_pysqlite_dbapiunicode_nocextensions 466
+
# TEST: test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set
test.aaa_profiling.test_orm.AttributeOverheadTest.test_attribute_set 2.7_mysql_mysqldb_dbapiunicode_cextensions 4256