--- /dev/null
+.. change::
+ :tags: orm, performance
+ :tickets: 5162
+
+ Modified the queries used by subqueryload and selectinload to no longer
+ ORDER BY the primary key of the parent entity; this ordering was there to
+ allow the rows as they come in to be copied into lists directly with a
+ minimal level of Python-side collation. However, these ORDER BY clauses
+ can negatively impact the performance of the query as in many scenarios
+ these columns are derived from a subquery or are otherwise not actual
+ primary key columns such that SQL planners cannot make use of indexes. The
+ Python-side collation uses the native itertools.group_by() to collate the
+ incoming rows, and has been modified to allow multiple
+ row-groups-per-parent to be assembled together using list.extend(), which
+ should still allow for relatively fast Python-side performance. There will
+ still be an ORDER BY present for a relationship that includes an explicit
+ order_by parameter, however this is the only ORDER BY that will be added to
+ the query for both kinds of loading.
addresses.email_address AS addresses_email_address
FROM addresses
WHERE addresses.user_id IN (?)
- ORDER BY addresses.user_id, addresses.id
+ ORDER BY addresses.id
(5,)
{stop}>>> jack
<User(name='jack', fullname='Jack Bean', nickname='gjffdd')>
to_join, local_attr, parent_alias = self._prep_for_joins(
left_alias, subq_path
)
- q = q.order_by(*local_attr)
+
q = q.add_columns(*local_attr)
q = self._apply_joins(
q, to_join, left_alias, parent_alias, effective_entity
return self._data.get(key, default)
def _load(self):
- self._data = dict(
- (k, [vv[0] for vv in v])
- for k, v in itertools.groupby(self.subq, lambda x: x[1:])
- )
+ self._data = collections.defaultdict(list)
+ for k, v in itertools.groupby(self.subq, lambda x: x[1:]):
+ self._data[k].extend(vv[0] for vv in v)
def loader(self, state, dict_, row):
if self._data is None:
q.add_criteria(
lambda q: q.filter(
in_expr.in_(sql.bindparam("primary_keys", expanding=True))
- ).order_by(*pk_cols)
+ )
)
orig_query = context.query
for key, state, state_dict, overwrite in chunk
]
- data = {
- k: [vv[1] for vv in v]
- for k, v in itertools.groupby(
- q(context.session).params(primary_keys=primary_keys),
- lambda x: x[0],
- )
- }
+ data = collections.defaultdict(list)
+ for k, v in itertools.groupby(
+ q(context.session).params(primary_keys=primary_keys),
+ lambda x: x[0],
+ ):
+ data[k].extend(vv[1] for vv in v)
for key, state, state_dict, overwrite in chunk:
"SELECT c.a_sub_id AS c_a_sub_id, "
"c.id AS c_id "
"FROM c WHERE c.a_sub_id "
- "IN ([POSTCOMPILE_primary_keys]) ORDER BY c.a_sub_id",
+ "IN ([POSTCOMPILE_primary_keys])",
{"primary_keys": [2]},
),
),
CompiledSQL(
"SELECT b.a_id AS b_a_id, b.id AS b_id FROM b "
- "WHERE b.a_id IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY b.a_id",
+ "WHERE b.a_id IN ([POSTCOMPILE_primary_keys])",
{"primary_keys": [1, 2]},
),
),
"people.name AS people_name, people.type AS people_type "
"FROM people WHERE people.company_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY people.company_id, people.person_id",
+ "ORDER BY people.person_id",
{"primary_keys": [1, 2]},
),
AllOf(
"JOIN managers ON people.person_id = managers.person_id) "
"AS pjoin WHERE pjoin.name = :name_1) AS anon_1 JOIN "
"machines ON anon_1.pjoin_person_id = machines.engineer_id "
- "ORDER BY anon_1.pjoin_person_id, machines.machine_id",
+ "ORDER BY machines.machine_id",
params=[{"name_1": "dilbert"}],
),
)
"employee.id AS employee_id FROM employee "
"WHERE employee.type IN ([POSTCOMPILE_type_1])) AS anon_1 "
"JOIN employee_stuff ON anon_1.employee_id "
- "= employee_stuff.employee_id ORDER BY "
- "anon_1.employee_id",
+ "= employee_stuff.employee_id",
use_default_dialect=True,
)
"SELECT a_1.id AS a_1_id, b.id AS b_id FROM a AS a_1 "
"JOIN (b JOIN d ON d.b_id = b.id JOIN c ON c.id = d.c_id) "
"ON a_1.b_id = b.id WHERE a_1.id "
- "IN ([POSTCOMPILE_primary_keys]) ORDER BY a_1.id",
+ "IN ([POSTCOMPILE_primary_keys])",
[{"primary_keys": [1]}],
),
)
"FROM (SELECT users.id AS "
"users_id FROM users WHERE users.id = :id_1) AS anon_1 "
"JOIN orders ON anon_1.users_id = orders.user_id ORDER BY "
- "anon_1.users_id, orders.id",
+ "orders.id",
[{"id_1": 7}],
),
],
"anon_1.t_a_id AS anon_1_t_a_id FROM "
"(SELECT t_a.id AS t_a_id FROM t_a) AS anon_1 "
"JOIN (t_b AS t_b_1 LEFT OUTER JOIN t_b2 AS t_b2_1 "
- "ON t_b_1.id = t_b2_1.id) ON anon_1.t_a_id = t_b_1.a_id "
- "ORDER BY anon_1.t_a_id",
+ "ON t_b_1.id = t_b2_1.id) ON anon_1.t_a_id = t_b_1.a_id",
{},
),
CompiledSQL(
"AS anon_1 JOIN (t_b AS t_b_1 LEFT OUTER JOIN t_b2 AS t_b2_1 "
"ON t_b_1.id = t_b2_1.id) ON anon_1.t_a_id = t_b_1.a_id "
"JOIN (t_c AS t_c_1 LEFT OUTER JOIN t_c2 AS t_c2_1 ON "
- "t_c_1.id = t_c2_1.id) ON t_b_1.id = t_c_1.b_id "
- "ORDER BY t_b_1.id",
+ "t_c_1.id = t_c2_1.id) ON t_b_1.id = t_c_1.b_id",
{},
),
CompiledSQL(
"JOIN (t_c AS t_c_1 LEFT OUTER JOIN t_c2 AS t_c2_1 "
"ON t_c_1.id = t_c2_1.id) "
"ON t_b_1.id = t_c_1.b_id "
- "JOIN t_d ON t_c_1.id = t_d.c_id ORDER BY t_c_1.id",
+ "JOIN t_d ON t_c_1.id = t_d.c_id",
{},
),
)
"(SELECT a.id AS aid, b.id AS id FROM a JOIN b ON a.b_ids "
"LIKE :id_1 || b.id || :param_1) AS anon_1 "
"ON a_1.id = anon_1.aid JOIN b ON b.id = anon_1.id "
- "WHERE a_1.id IN ([POSTCOMPILE_primary_keys]) ORDER BY a_1.id",
+ "WHERE a_1.id IN ([POSTCOMPILE_primary_keys])",
params=[{"id_1": "%", "param_1": "%", "primary_keys": [2]}],
),
)
"paperwork.description AS paperwork_description "
"FROM paperwork WHERE paperwork.person_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY paperwork.person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
[{"primary_keys": [1]}],
),
)
"paperwork.description AS paperwork_description "
"FROM paperwork WHERE paperwork.person_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY paperwork.person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
[{"primary_keys": [1]}],
),
)
"paperwork.description AS paperwork_description "
"FROM paperwork WHERE paperwork.person_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY paperwork.person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
[{"primary_keys": [1]}],
),
)
"paperwork.description AS paperwork_description "
"FROM paperwork WHERE paperwork.person_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY paperwork.person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
[{"primary_keys": [1]}],
),
)
"paperwork.description AS paperwork_description "
"FROM paperwork WHERE paperwork.person_id "
"IN ([POSTCOMPILE_primary_keys]) "
- "ORDER BY paperwork.person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
[{"primary_keys": [1]}],
),
)
CompiledSQL(
"SELECT b.a_id1 AS b_a_id1, b.a_id2 AS b_a_id2, b.id AS b_id "
"FROM b WHERE (b.a_id1, b.a_id2) IN "
- "([POSTCOMPILE_primary_keys]) ORDER BY b.a_id1, b.a_id2, b.id",
+ "([POSTCOMPILE_primary_keys]) ORDER BY b.id",
[{"primary_keys": [(i, i + 2) for i in range(1, 20)]}],
),
)
CompiledSQL(
"SELECT b.a_id AS b_a_id, b.id AS b_id "
"FROM b WHERE b.a_id IN "
- "([POSTCOMPILE_primary_keys]) ORDER BY b.a_id, b.id",
+ "([POSTCOMPILE_primary_keys]) ORDER BY b.id",
{"primary_keys": list(range(1, 48))},
),
CompiledSQL(
"SELECT b.a_id AS b_a_id, b.id AS b_id "
"FROM b WHERE b.a_id IN "
- "([POSTCOMPILE_primary_keys]) ORDER BY b.a_id, b.id",
+ "([POSTCOMPILE_primary_keys]) ORDER BY b.id",
{"primary_keys": list(range(48, 95))},
),
CompiledSQL(
"SELECT b.a_id AS b_a_id, b.id AS b_id "
"FROM b WHERE b.a_id IN "
- "([POSTCOMPILE_primary_keys]) ORDER BY b.a_id, b.id",
+ "([POSTCOMPILE_primary_keys]) ORDER BY b.id",
{"primary_keys": list(range(95, 101))},
),
)
CompiledSQL(
"SELECT role.user_id AS role_user_id, role.id AS role_id "
"FROM role WHERE role.user_id "
- "IN ([POSTCOMPILE_primary_keys]) ORDER BY role.user_id",
+ "IN ([POSTCOMPILE_primary_keys])",
{"primary_keys": [1]},
),
)
"SELECT a_1.id AS a_1_id, b.id AS b_id, b.x AS b_x, "
"b.y AS b_y "
"FROM a AS a_1 JOIN b ON b.id = a_1.b_id "
- "WHERE a_1.id IN ([POSTCOMPILE_primary_keys]) ORDER BY a_1.id",
+ "WHERE a_1.id IN ([POSTCOMPILE_primary_keys])",
[{"primary_keys": [1, 3]}],
),
)
CompiledSQL(
"SELECT a_1.id AS a_1_id, b.id AS b_id, b.x AS b_x, "
"b.y AS b_y FROM a AS a_1 JOIN b ON b.id = a_1.b_id "
- "WHERE a_1.id IN ([POSTCOMPILE_primary_keys]) ORDER BY a_1.id",
+ "WHERE a_1.id IN ([POSTCOMPILE_primary_keys])",
[{"primary_keys": [1, 2, 3, 4, 5]}],
),
)
"SELECT a_1.id AS a_1_id, b.id AS b_id, b.x AS b_x, "
"b.y AS b_y "
"FROM a AS a_1 JOIN b ON b.id = a_1.b_id "
- "WHERE a_1.id IN ([POSTCOMPILE_primary_keys]) ORDER BY a_1.id",
+ "WHERE a_1.id IN ([POSTCOMPILE_primary_keys])",
[{"primary_keys": [1, 2, 3, 4, 5]}],
),
)
":primary_language_1) AS anon_1 "
"JOIN paperwork "
"ON anon_1.people_person_id = paperwork.person_id "
- "ORDER BY anon_1.people_person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
{"primary_language_1": "java"},
),
)
"paperwork.description = :description_1) AS anon_1 "
"JOIN paperwork ON anon_1.people_person_id = "
"paperwork.person_id "
- "ORDER BY anon_1.people_person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
{
"primary_language_1": "java",
"description_1": "tps report #2",
"WHERE engineers.primary_language = :primary_language_1) "
"AS anon_1 JOIN paperwork "
"ON anon_1.people_person_id = paperwork.person_id "
- "ORDER BY anon_1.people_person_id, paperwork.paperwork_id",
+ "ORDER BY paperwork.paperwork_id",
{"primary_language_1": "java"},
),
CompiledSQL(
"AS anon_1 JOIN paperwork AS paperwork_1 "
"ON anon_1.people_person_id = paperwork_1.person_id "
"JOIN pages ON paperwork_1.paperwork_id = pages.paperwork_id "
- "ORDER BY paperwork_1.paperwork_id, pages.page_id",
+ "ORDER BY pages.page_id",
{"primary_language_1": "java"},
),
)
"engineers.engineer_id ORDER BY engineers.primary_language "
"DESC LIMIT :param_1) AS anon_1 JOIN paperwork "
"ON anon_1.people_person_id = paperwork.person_id "
- "ORDER BY anon_1.people_person_id, paperwork.paperwork_id"
+ "ORDER BY paperwork.paperwork_id"
),
)
"DESC LIMIT :param_1) AS anon_1 "
"JOIN paperwork "
"ON anon_1.anon_2_people_person_id = paperwork.person_id "
- "ORDER BY anon_1.anon_2_people_person_id, "
- "paperwork.paperwork_id"
+ "ORDER BY paperwork.paperwork_id"
),
)
"ON people_1.person_id = engineers_1.engineer_id "
"ORDER BY engineers_1.primary_language DESC LIMIT :param_1) "
"AS anon_1 JOIN paperwork ON anon_1.people_1_person_id = "
- "paperwork.person_id ORDER BY anon_1.people_1_person_id, "
- "paperwork.paperwork_id"
+ "paperwork.person_id ORDER BY paperwork.paperwork_id"
),
)
"ON persistent.id = director.id) AS anon_1 "
"JOIN (persistent JOIN movie "
"ON persistent.id = movie.id) "
- "ON anon_1.director_id = movie.director_id "
- "ORDER BY anon_1.director_id",
+ "ON anon_1.director_id = movie.director_id",
dialect="default",
)
"anon_1.movie_director_id AS anon_1_movie_director_id "
"FROM (SELECT%s movie.director_id AS movie_director_id "
"FROM movie) AS anon_1 "
- "JOIN director ON director.id = anon_1.movie_director_id "
- "ORDER BY anon_1.movie_director_id"
+ "JOIN director ON director.id = anon_1.movie_director_id"
% (" DISTINCT" if expect_distinct else ""),
)
"JOIN director AS director_1 "
"ON director_1.id = anon_1.movie_director_id "
"JOIN director_photo "
- "ON director_1.id = director_photo.director_id "
- "ORDER BY director_1.id"
+ "ON director_1.id = director_photo.director_id"
% (" DISTINCT" if expect_distinct else ""),
)
result = s.execute(q3)
"anon_1.foo_foo_id AS anon_1_foo_foo_id "
"FROM (SELECT DISTINCT foo.foo_id AS foo_foo_id "
"FROM foo WHERE foo.id = :id_1) AS anon_1 "
- "JOIN foo AS foo_1 ON foo_1.id = anon_1.foo_foo_id "
- "ORDER BY anon_1.foo_foo_id",
+ "JOIN foo AS foo_1 ON foo_1.id = anon_1.foo_foo_id",
{"id_1": 2},
),
CompiledSQL(
"FROM (SELECT DISTINCT foo.foo_id AS foo_foo_id FROM foo "
"WHERE foo.id = :id_1) AS anon_1 "
"JOIN foo AS foo_1 ON foo_1.id = anon_1.foo_foo_id "
- "JOIN foo ON foo.id = foo_1.foo_id ORDER BY foo_1.foo_id",
+ "JOIN foo ON foo.id = foo_1.foo_id",
{"id_1": 2},
),
)
projects / thirdparty / sqlalchemy / sqlalchemy.git / commitdiff
