From: Mike Bayer Date: Wed, 23 Aug 2017 20:33:10 +0000 (-0400) Subject: Ensure col is not None when retrieving quick populators X-Git-Tag: rel_1_1_14~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3b5840f16a27e972c950890b3bbab1d9bfc460a6;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Ensure col is not None when retrieving quick populators Fixed bug where an :func:`.undefer_group` option would not be recognized if it extended from a relationship that was loading using joined eager loading. In particular we need to double check the column both in terms of the given "adapter" as well as without applying the "adapter" when searching for the column in the result. As we now avoid redoing the row processor step we also improve on callcounts in joined eager loading. Change-Id: I0f48766f12f7299f4626ff41a00bf1f5bfca5f3b Fixes: #4048 (cherry picked from commit eee9dfd4514801f0c67f71632fc722731171479b) --- diff --git a/doc/build/changelog/unreleased_11/4048.rst b/doc/build/changelog/unreleased_11/4048.rst new file mode 100644 index 0000000000..612736195f --- /dev/null +++ b/doc/build/changelog/unreleased_11/4048.rst @@ -0,0 +1,11 @@ +.. change:: + :tags: bug, orm + :tickets: 4048 + :versions: 1.2.0b3 + + Fixed bug where an :func:`.undefer_group` option would not be recognized + if it extended from a relationship that was loading using joined eager + loading. Additionally, as the bug led to excess work being performed, + Python function call counts are also improved by 20% within the initial + calculation of result set columns, complementing the joined eager load + improvements of :ticket:`3915`. diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py index f749cdd1db..6f0c860cc5 100644 --- a/lib/sqlalchemy/orm/loading.py +++ b/lib/sqlalchemy/orm/loading.py @@ -314,9 +314,24 @@ def _instance_processor( # be present in some unexpected way. populators["expire"].append((prop.key, False)) else: + getter = None + # the "adapter" can be here via different paths, + # e.g. via adapter present at setup_query or adapter + # applied to the query afterwards via eager load subquery. + # If the column here + # were already a product of this adapter, sending it through + # the adapter again can return a totally new expression that + # won't be recognized in the result, and the ColumnAdapter + # currently does not accommodate for this. OTOH, if the + # column were never applied through this adapter, we may get + # None back, in which case we still won't get our "getter". + # so try both against result._getter(). See issue #4048 if adapter: - col = adapter.columns[col] - getter = result._getter(col, False) + adapted_col = adapter.columns[col] + if adapted_col is not None: + getter = result._getter(adapted_col, False) + if not getter: + getter = result._getter(col, False) if getter: populators["quick"].append((prop.key, getter)) else: diff --git a/test/orm/test_deferred.py b/test/orm/test_deferred.py index 4b5eafffed..7f3273bb52 100644 --- a/test/orm/test_deferred.py +++ b/test/orm/test_deferred.py @@ -2,7 +2,8 @@ import sqlalchemy as sa from sqlalchemy import testing, util from sqlalchemy.orm import mapper, deferred, defer, undefer, Load, \ load_only, undefer_group, create_session, synonym, relationship, Session,\ - joinedload, defaultload, aliased, contains_eager, with_polymorphic + joinedload, defaultload, aliased, contains_eager, with_polymorphic, \ + subqueryload from sqlalchemy.testing import eq_, AssertsCompiledSQL, assert_raises_message from test.orm import _fixtures @@ -383,6 +384,167 @@ class DeferredOptionsTest(AssertsCompiledSQL, _fixtures.FixtureTest): "FROM orders ORDER BY orders.id", {})]) + def test_undefer_group_from_relationship_lazyload(self): + users, Order, User, orders = \ + (self.tables.users, + self.classes.Order, + self.classes.User, + self.tables.orders) + + mapper(User, users, properties=dict( + orders=relationship(Order, order_by=orders.c.id))) + mapper( + Order, orders, properties=util.OrderedDict([ + ('userident', deferred(orders.c.user_id, group='primary')), + ('description', deferred(orders.c.description, + group='primary')), + ('opened', deferred(orders.c.isopen, group='primary')) + ]) + ) + + sess = create_session() + q = sess.query(User).filter(User.id == 7).options( + defaultload(User.orders).undefer_group('primary') + ) + + def go(): + result = q.all() + o2 = result[0].orders[1] + eq_(o2.opened, 1) + eq_(o2.userident, 7) + eq_(o2.description, 'order 3') + self.sql_eq_(go, [ + ("SELECT users.id AS users_id, users.name AS users_name " + "FROM users WHERE users.id = :id_1", {"id_1": 7}), + ("SELECT orders.user_id AS orders_user_id, orders.description " + "AS orders_description, orders.isopen AS orders_isopen, " + "orders.id AS orders_id, orders.address_id AS orders_address_id " + "FROM orders WHERE :param_1 = orders.user_id ORDER BY orders.id", + {'param_1': 7})]) + + def test_undefer_group_from_relationship_subqueryload(self): + users, Order, User, orders = \ + (self.tables.users, + self.classes.Order, + self.classes.User, + self.tables.orders) + + mapper(User, users, properties=dict( + orders=relationship(Order, order_by=orders.c.id))) + mapper( + Order, orders, properties=util.OrderedDict([ + ('userident', deferred(orders.c.user_id, group='primary')), + ('description', deferred(orders.c.description, + group='primary')), + ('opened', deferred(orders.c.isopen, group='primary')) + ]) + ) + + sess = create_session() + q = sess.query(User).filter(User.id == 7).options( + subqueryload(User.orders).undefer_group('primary') + ) + + def go(): + result = q.all() + o2 = result[0].orders[1] + eq_(o2.opened, 1) + eq_(o2.userident, 7) + eq_(o2.description, 'order 3') + self.sql_eq_(go, [ + ("SELECT users.id AS users_id, users.name AS users_name " + "FROM users WHERE users.id = :id_1", {"id_1": 7}), + ("SELECT orders.user_id AS orders_user_id, orders.description " + "AS orders_description, orders.isopen AS orders_isopen, " + "orders.id AS orders_id, orders.address_id AS orders_address_id, " + "anon_1.users_id AS anon_1_users_id FROM (SELECT users.id AS " + "users_id FROM users WHERE users.id = :id_1) AS anon_1 " + "JOIN orders ON anon_1.users_id = orders.user_id ORDER BY " + "anon_1.users_id, orders.id", [{'id_1': 7}])] + ) + + def test_undefer_group_from_relationship_joinedload(self): + users, Order, User, orders = \ + (self.tables.users, + self.classes.Order, + self.classes.User, + self.tables.orders) + + mapper(User, users, properties=dict( + orders=relationship(Order, order_by=orders.c.id))) + mapper( + Order, orders, properties=util.OrderedDict([ + ('userident', deferred(orders.c.user_id, group='primary')), + ('description', deferred(orders.c.description, + group='primary')), + ('opened', deferred(orders.c.isopen, group='primary')) + ]) + ) + + sess = create_session() + q = sess.query(User).filter(User.id == 7).options( + joinedload(User.orders).undefer_group('primary') + ) + + def go(): + result = q.all() + o2 = result[0].orders[1] + eq_(o2.opened, 1) + eq_(o2.userident, 7) + eq_(o2.description, 'order 3') + self.sql_eq_(go, [ + ("SELECT users.id AS users_id, users.name AS users_name, " + "orders_1.user_id AS orders_1_user_id, orders_1.description AS " + "orders_1_description, orders_1.isopen AS orders_1_isopen, " + "orders_1.id AS orders_1_id, orders_1.address_id AS " + "orders_1_address_id FROM users " + "LEFT OUTER JOIN orders AS orders_1 ON users.id = " + "orders_1.user_id WHERE users.id = :id_1 " + "ORDER BY orders_1.id", {"id_1": 7})] + ) + + def test_undefer_group_from_relationship_joinedload_colexpr(self): + users, Order, User, orders = \ + (self.tables.users, + self.classes.Order, + self.classes.User, + self.tables.orders) + + mapper(User, users, properties=dict( + orders=relationship(Order, order_by=orders.c.id))) + mapper( + Order, orders, properties=util.OrderedDict([ + ('userident', deferred(orders.c.user_id, group='primary')), + ('lower_desc', deferred( + sa.func.lower(orders.c.description).label(None), + group='primary')), + ('opened', deferred(orders.c.isopen, group='primary')) + ]) + ) + + sess = create_session() + q = sess.query(User).filter(User.id == 7).options( + joinedload(User.orders).undefer_group('primary') + ) + + def go(): + result = q.all() + o2 = result[0].orders[1] + eq_(o2.opened, 1) + eq_(o2.userident, 7) + eq_(o2.lower_desc, 'order 3') + self.sql_eq_(go, [ + ("SELECT users.id AS users_id, users.name AS users_name, " + "orders_1.user_id AS orders_1_user_id, " + "lower(orders_1.description) AS lower_1, " + "orders_1.isopen AS orders_1_isopen, orders_1.id AS orders_1_id, " + "orders_1.address_id AS orders_1_address_id, " + "orders_1.description AS orders_1_description FROM users " + "LEFT OUTER JOIN orders AS orders_1 ON users.id = " + "orders_1.user_id WHERE users.id = :id_1 " + "ORDER BY orders_1.id", {"id_1": 7})] + ) + def test_undefer_star(self): orders, Order = self.tables.orders, self.classes.Order diff --git a/test/profiles.txt b/test/profiles.txt index e056f05325..8fca44c008 100644 --- a/test/profiles.txt +++ b/test/profiles.txt @@ -1,15 +1,15 @@ # /home/classic/dev/sqlalchemy/test/profiles.txt # This file is written out on a per-environment basis. -# For each test in aaa_profiling, the corresponding function and +# For each test in aaa_profiling, the corresponding function and # environment is located within this file. If it doesn't exist, # the test is skipped. -# If a callcount does exist, it is compared to what we received. +# If a callcount does exist, it is compared to what we received. # assertions are raised if the counts do not match. -# -# To add a new callcount test, apply the function_call_count -# decorator and re-run the tests using the --write-profiles +# +# To add a new callcount test, apply the function_call_count +# decorator and re-run the tests using the --write-profiles # option - this file will be rewritten including the new count. -# +# # TEST: test.aaa_profiling.test_compiler.CompileTest.test_insert @@ -183,18 +183,18 @@ test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_build_query 3.5_sqlite_pysq # TEST: test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_mysql_mysqldb_dbapiunicode_cextensions 567666 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_mysql_mysqldb_dbapiunicode_nocextensions 580466 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_postgresql_psycopg2_dbapiunicode_cextensions 578324 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 591124 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_sqlite_pysqlite_dbapiunicode_cextensions 557938 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 570738 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_mysql_mysqldb_dbapiunicode_cextensions 570849 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_mysql_mysqldb_dbapiunicode_nocextensions 583849 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_postgresql_psycopg2_dbapiunicode_cextensions 598107 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_postgresql_psycopg2_dbapiunicode_nocextensions 611107 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_sqlite_pysqlite_dbapiunicode_cextensions 562621 -test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_sqlite_pysqlite_dbapiunicode_nocextensions 575621 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_mysql_mysqldb_dbapiunicode_cextensions 430866 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_mysql_mysqldb_dbapiunicode_nocextensions 443666 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_postgresql_psycopg2_dbapiunicode_cextensions 441526 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_postgresql_psycopg2_dbapiunicode_nocextensions 454326 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_sqlite_pysqlite_dbapiunicode_cextensions 421140 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 2.7_sqlite_pysqlite_dbapiunicode_nocextensions 433940 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_mysql_mysqldb_dbapiunicode_cextensions 434049 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_mysql_mysqldb_dbapiunicode_nocextensions 447049 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_postgresql_psycopg2_dbapiunicode_cextensions 461309 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_postgresql_psycopg2_dbapiunicode_nocextensions 474309 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_sqlite_pysqlite_dbapiunicode_cextensions 425823 +test.aaa_profiling.test_orm.JoinedEagerLoadTest.test_fetch_results 3.5_sqlite_pysqlite_dbapiunicode_nocextensions 438823 # TEST: test.aaa_profiling.test_orm.LoadManyToOneFromIdentityTest.test_many_to_one_load_identity