From 70fbce17cda3fd9ea36febae6b93c3c5877f367f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 8 Jun 2022 13:05:20 -0400 Subject: [PATCH] suppport with_loader_criteria pickling w/ fixed callable Fixed issue where a :func:`_orm.with_loader_criteria` option could not be pickled, as is necessary when it is carried along for propagation to lazy loaders in conjunction with a caching scheme. Currently, the only form that is supported as picklable is to pass the "where criteria" as a fixed module-level callable function that produces a SQL expression. An ad-hoc "lambda" can't be pickled, and a SQL expression object is usually not fully picklable directly. Fixes: #8109 Change-Id: I49fe69088b0c7e58a0f22c67d2ea4e33752a5a73 (cherry picked from commit 293b0e3dd8205185b84cd3baf2f078348437d245) --- doc/build/changelog/unreleased_14/8109.rst | 12 ++++++ lib/sqlalchemy/orm/util.py | 33 ++++++++++++++-- test/orm/test_pickled.py | 45 ++++++++++++++++++++++ 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 doc/build/changelog/unreleased_14/8109.rst diff --git a/doc/build/changelog/unreleased_14/8109.rst b/doc/build/changelog/unreleased_14/8109.rst new file mode 100644 index 0000000000..cf64d21ac4 --- /dev/null +++ b/doc/build/changelog/unreleased_14/8109.rst @@ -0,0 +1,12 @@ +.. change:: + :tags: bug, orm + :tickets: 8109 + + Fixed issue where a :func:`_orm.with_loader_criteria` option could not be + pickled, as is necessary when it is carried along for propagation to lazy + loaders in conjunction with a caching scheme. Currently, the only form that + is supported as picklable is to pass the "where criteria" as a fixed + module-level callable function that produces a SQL expression. An ad-hoc + "lambda" can't be pickled, and a SQL expression object is usually not fully + picklable directly. + diff --git a/lib/sqlalchemy/orm/util.py b/lib/sqlalchemy/orm/util.py index 1c9c9c796f..66f42ba810 100644 --- a/lib/sqlalchemy/orm/util.py +++ b/lib/sqlalchemy/orm/util.py @@ -907,6 +907,8 @@ class _WrapUserEntity(object): """ + __slots__ = ("subject",) + def __init__(self, subject): self.subject = subject @@ -1092,10 +1094,9 @@ class LoaderCriteriaOption(CriteriaOption): accepts a target class as an argument, when the given class is a base with many different mapped subclasses. - .. note:: when the SQL expression is a lambda, **pickling is not - supported**. Set - :paramref:`_orm.with_loader_criteria.propagate_to_loaders` - to ``False`` to prevent the object from being applied to instances. + .. note:: To support pickling, use a module-level Python function to + produce the SQL expression instead of a lambda or a fixed SQL + expression, which tend to not be picklable. :param include_aliases: if True, apply the rule to :func:`_orm.aliased` constructs as well. @@ -1132,6 +1133,7 @@ class LoaderCriteriaOption(CriteriaOption): self.root_entity = None self.entity = entity + self._where_crit_orig = where_criteria if callable(where_criteria): self.deferred_where_criteria = True self.where_criteria = lambdas.DeferredLambdaElement( @@ -1157,7 +1159,30 @@ class LoaderCriteriaOption(CriteriaOption): self.include_aliases = include_aliases self.propagate_to_loaders = propagate_to_loaders + @classmethod + def _unreduce( + cls, entity, where_criteria, include_aliases, propagate_to_loaders + ): + return LoaderCriteriaOption( + entity, + where_criteria, + include_aliases=include_aliases, + propagate_to_loaders=propagate_to_loaders, + ) + + def __reduce__(self): + return ( + LoaderCriteriaOption._unreduce, + ( + self.entity.class_ if self.entity else None, + self._where_crit_orig, + self.include_aliases, + self.propagate_to_loaders, + ), + ) + def _all_mappers(self): + if self.entity: for ent in self.entity.mapper.self_and_descendants: yield ent diff --git a/test/orm/test_pickled.py b/test/orm/test_pickled.py index 11d90bd590..c1be0ca25c 100644 --- a/test/orm/test_pickled.py +++ b/test/orm/test_pickled.py @@ -15,6 +15,7 @@ from sqlalchemy.orm import lazyload from sqlalchemy.orm import relationship from sqlalchemy.orm import state as sa_state from sqlalchemy.orm import subqueryload +from sqlalchemy.orm import with_loader_criteria from sqlalchemy.orm import with_polymorphic from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.orm.collections import column_mapped_collection @@ -43,6 +44,10 @@ from .inheritance._poly_fixtures import Manager from .inheritance._poly_fixtures import Person +def no_ed_foo(cls): + return cls.email_address != "ed@foo.com" + + class PickleTest(fixtures.MappedTest): @classmethod def define_tables(cls, metadata): @@ -324,6 +329,46 @@ class PickleTest(fixtures.MappedTest): u2.addresses.append(Address()) eq_(len(u2.addresses), 2) + @testing.requires.python3 + @testing.combinations(True, False, argnames="pickle_it") + def test_loader_criteria(self, pickle_it): + """test #8109""" + + users, addresses = (self.tables.users, self.tables.addresses) + + self.mapper_registry.map_imperatively( + User, + users, + properties={"addresses": relationship(Address)}, + ) + self.mapper_registry.map_imperatively(Address, addresses) + + with fixture_session(expire_on_commit=False) as sess: + u1 = User(name="ed") + u1.addresses = [ + Address(email_address="ed@bar.com"), + Address(email_address="ed@foo.com"), + ] + sess.add(u1) + sess.commit() + + with fixture_session(expire_on_commit=False) as sess: + # note that non-lambda is not picklable right now as + # SQL expressions usually can't be pickled. + opt = with_loader_criteria( + Address, + no_ed_foo, + ) + + u1 = sess.query(User).options(opt).first() + + if pickle_it: + u1 = pickle.loads(pickle.dumps(u1)) + sess.close() + sess.add(u1) + + eq_([ad.email_address for ad in u1.addresses], ["ed@bar.com"]) + @testing.requires.non_broken_pickle def test_instance_deferred_cols(self): users, addresses = (self.tables.users, self.tables.addresses) -- 2.47.2